public async Task ProcessEvents_InvalidDocument_AndUnknownDocumentType_AbandonsInvalidAndUnknownDocuments()
{
_client.SimulateElasticSearch();
var events = new[]
{
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateSerilogEventData(@"invalid json"),
MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
MockEventData.CreateSerilogEventData(@""),
MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"),
MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once for the initial request, MaxFailedDocumentRetries times to retry the failed documents,
// once to abandon the invalid documents, and once to abandon the unknown document type.
_client.VerifyBulkAsyncCalled(Times.Exactly(1 + MaxFailedDocumentRetries + 1 + 1));
Assert.Equal(4 + 2 + 3, _client.GetTotalItems()); // 4 successful, 2 ingestionstats, 3 abandoned
Assert.Equal(1, _client.GetTotalItemsInIndex("logstash"));
Assert.Equal(1, _client.GetTotalItemsInIndex("robointeractions"));
Assert.Equal(1, _client.GetTotalItemsInIndex("externaltelemetry"));
Assert.Equal(1, _client.GetTotalItemsInIndex("azure-resources"));
Assert.Equal(2, _client.GetTotalItemsInIndex("ingestionstats"));
Assert.Equal(3, _client.GetTotalItemsInIndex("abandoneddocs"));
Assert.Equal(2, _processor.LastBatchFailedDocuments);
Assert.Equal(3, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument()
{
_client.SimulateElasticSearch();
var events = new[]
{
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateSerilogEventData(@"
{
""message"": ""documents formatted with newlines are invalid for Elasticsearch bulk upload."",
""data"": 2
}" ),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 3}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once for the initial request, then once to abandon the invalid document.
_client.VerifyBulkAsyncCalled(Times.Exactly(1 + 1));
Assert.Equal(0, _processor.LastBatchFailedDocuments);
Assert.Equal(1, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_MultipleDocumentTypes_Succeeds()
{
_client.SimulateElasticSearch();
var events = new[]
{
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"),
MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once for the initial request.
_client.VerifyBulkAsyncCalled(Times.Exactly(1));
Assert.Equal(4 + 2, _client.GetTotalItems()); // 4 for serilog/robocustos/externaltelemetry/azure-resources indexes, 2 for ingestionstats
Assert.Equal(1, _client.GetTotalItemsInIndex("logstash"));
Assert.Equal(1, _client.GetTotalItemsInIndex("robointeractions"));
Assert.Equal(1, _client.GetTotalItemsInIndex("externaltelemetry"));
Assert.Equal(1, _client.GetTotalItemsInIndex("azure-resources"));
Assert.Equal(2, _client.GetTotalItemsInIndex("ingestionstats"));
Assert.Equal(0, _processor.LastBatchFailedDocuments);
Assert.Equal(0, _processor.LastBatchAbandonedDocuments);
}
public void ToBulkString_MultilineContent_MultipleNewlines()
{
var data = MockEventData.CreateSerilogEventData(body: "{ \"message\": \"log message\n with newline\" }");
var item = BulkItem.FromEventData(data);
var bulkBody = item.ToBulkString();
Assert.False(item.IsValid, "BulkItem with document body containing newlines should be invalid.");
Assert.Equal(2, bulkBody.Count(c => c == '\n'));
}
public void ToBulkString_SimpleContent_OneNewline()
{
var data = MockEventData.CreateSerilogEventData(body: "{ \"message\": \"log message\" }");
var item = BulkItem.FromEventData(data);
var bulkBody = item.ToBulkString();
// BulkString should have exactly one newline when formatted correctly.
Assert.Equal(1, bulkBody.Count(c => c == '\n'));
}
public void ToBulkString_ComplexContent_OneNewline()
{
var data = MockEventData.CreateSerilogEventData(body: JsonConvert.SerializeObject(new { message = "log message\n with newline" }));
var item = BulkItem.FromEventData(data);
var bulkBody = item.ToBulkString();
// BulkString should have exactly one newline when formatted correctly.
Assert.True(item.IsValid, "BulkItem with json encoded newlines should be valid.");
Assert.Equal(1, bulkBody.Count(c => c == '\n'));
}
public async Task ProcessEvents_OneEmptyDocument_AbandonsDocument()
{
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, new[] { MockEventData.CreateSerilogEventData("") });
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(1, _processor.LastBatchFailedDocuments);
Assert.Equal(1, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneDocument_Succeeds_OneElasticClientCall()
{
_client.OnBulkAsync().Returns(HttpStatusCode.OK);
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, new [] { MockEventData.CreateSerilogEventData("{}") });
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
_client.VerifyBulkAsyncCalled(Times.Once());
Assert.Equal(0, _processor.LastBatchFailedDocuments);
Assert.Equal(0, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneEmptyDocument_AbandonsDocument()
{
_client.SimulateElasticSearch();
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, new[] { MockEventData.CreateSerilogEventData("") });
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once for the initial request, MaxFailedDocumentRetries times to retry the failed document, then once to abandon the document.
_client.VerifyBulkAsyncCalled(Times.Exactly(1 + MaxFailedDocumentRetries + 1));
Assert.Equal(1, _processor.LastBatchFailedDocuments);
Assert.Equal(1, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneDocument_Succeeds()
{
var events = new[]
{
MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_OneDocument_Succeeds\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 1 }}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(0, _processor.LastBatchFailedDocuments);
Assert.Equal(0, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneUnknownDocumentType_TwoValidDocuments_AbandonsUnknownDocumentType()
{
var events = new[]
{
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(0, _processor.LastBatchFailedDocuments); // The invalid document type should not be retried.
Assert.Equal(1, _processor.LastBatchAbandonedDocuments); // The invalid document type should be abandoned immediately.
}
public async Task ProcessEvents_MultipleInvalidDocuments_AbandonsInvalidDocuments()
{
var events = new[]
{
MockEventData.CreateSerilogEventData(@"invalid json"),
MockEventData.CreateSerilogEventData(@""),
MockEventData.CreateSerilogEventData(@"invalid json"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(3, _processor.LastBatchFailedDocuments);
Assert.Equal(3, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument()
{
var events = new[]
{
MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 1 }}"),
MockEventData.CreateSerilogEventData(@"invalid json"),
MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 2 }}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(1, _processor.LastBatchFailedDocuments);
Assert.Equal(1, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneDocument_WithFailures_RetriesElasticClientCallUntilSuccess()
{
// Fail the BulkAsync call 100 times with BadGateway, then succeed with OK.
const int failureCount = 100;
int callCount = 0;
_client.OnBulkAsync().Returns(() => ElasticsearchResponseExtensions.GetBulkResponse(++callCount <= failureCount ? HttpStatusCode.BadGateway : HttpStatusCode.OK));
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, new[] { MockEventData.CreateSerilogEventData("{}") });
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
_client.VerifyBulkAsyncCalled(Times.Exactly(failureCount + 1));
Assert.Equal(0, _processor.LastBatchFailedDocuments);
Assert.Equal(0, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_MultipleDocumentTypes_Succeeds()
{
var events = new[]
{
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"),
MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(0, _processor.LastBatchFailedDocuments);
Assert.Equal(0, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument()
{
var events = new[]
{
MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 1 }}"),
MockEventData.CreateSerilogEventData(@"
{
""message"": ""documents formatted with newlines are invalid for Elasticsearch bulk upload."",
""data"": 2
}" ),
MockEventData.CreateSerilogEventData($"{{ \"message\": \"ProcessEvents_DocumentWithNewlines_AbandonsInvalidDocument\", \"@timestamp\":\"{DateTime.UtcNow:o}\", \"data\": 3 }}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(0, _processor.LastBatchFailedDocuments);
Assert.Equal(1, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_InvalidDocument_AndUnknownDocumentType_AbandonsInvalidAndUnknownDocuments()
{
var events = new[]
{
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateSerilogEventData(@"invalid json"),
MockEventData.CreateRoboCustosEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
MockEventData.CreateSerilogEventData(@""),
MockEventData.CreateExternalTelemetryEventData(@"{ ""message"": ""log message"", ""data"": 3}"),
MockEventData.CreateAzureResourcesEventData(@"{ ""message"": ""log message"", ""data"": 4}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
Assert.Equal(2, _processor.LastBatchFailedDocuments);
Assert.Equal(3, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneInvalidJsonDocument_TwoValidDocuments_AbandonsInvalidJsonDocument()
{
_client.SimulateElasticSearch();
var events = new[]
{
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateSerilogEventData(@"invalid json"),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once for the initial request, MaxFailedDocumentRetries times to retry the failed documents, then once to abandon the document.
_client.VerifyBulkAsyncCalled(Times.Exactly(1 + MaxFailedDocumentRetries + 1));
Assert.Equal(1, _processor.LastBatchFailedDocuments);
Assert.Equal(1, _processor.LastBatchAbandonedDocuments);
}
public async Task ProcessEvents_OneUnknownDocumentType_TwoValidDocuments_AbandonsUnknownDocumentType()
{
_client.SimulateElasticSearch();
var events = new[]
{
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 1}"),
MockEventData.CreateInvalidEventData(@"{ ""message"": ""invalid type"" }"),
MockEventData.CreateSerilogEventData(@"{ ""message"": ""log message"", ""data"": 2}"),
};
await _processor.OpenAsync(_partition);
await _processor.ProcessEventsAsync(_partition, events);
await _processor.CloseAsync(_partition, CloseReason.Shutdown);
// We should try calling BulkAsync once for the initial request, then once to abandon the invalid document.
_client.VerifyBulkAsyncCalled(Times.Exactly(1 + 1));
Assert.Equal(0, _processor.LastBatchFailedDocuments); // The invalid document type should not be retried.
Assert.Equal(1, _processor.LastBatchAbandonedDocuments); // The invalid document type should be abandoned immediately.
}