private BulkItem GetBatchStats(
PartitionContext context,
DateTime now,
int messageCount,
BulkItem lastItem,
BulkItem oldestItemByTimestamp,
BulkItem oldestItemByEnqueueTime,
TimeSpan lag,
TimeSpan maxLag,
TimeSpan lagFromMessageCreation,
TimeSpan maxLagFromMessageCreation)
{
var messageBody =
"{" +
$"\"lastMessageTimestampInBatch\":\"{lastItem.Timestamp:o}\"," +
$"\"lastMessageEnqueueTimeInBatch\":\"{lastItem.EnqueueTime:o}\"," +
$"\"oldestMessageTimestampInBatch\":\"{oldestItemByTimestamp.Timestamp:o}\"," +
$"\"oldestMessageEnqueueTimeInBatch\":\"{oldestItemByEnqueueTime.EnqueueTime:o}\"," +
$"\"idOfOldestMessageInBatch\":\"{oldestItemByTimestamp.DocumentId}\"," +
$"\"idOfOldestEnqueuedMessageInBatch\":\"{oldestItemByEnqueueTime.DocumentId}\"," +
$"\"lagInMilliseconds\":{lag.TotalMilliseconds}," +
$"\"maxLagInMilliseconds\":{maxLag.TotalMilliseconds}," +
$"\"lagInMinutes\":{lag.TotalMinutes}," +
$"\"maxLagInMinutes\":{maxLag.TotalMinutes}," +
$"\"lagFromMessageCreationTimeInMinutes\":{lagFromMessageCreation.TotalMinutes}," +
$"\"maxLagFromMessageCreationTimeInMinutes\":{maxLagFromMessageCreation.TotalMinutes}," +
$"\"timestamp\":\"{now:o}\"," +
$"\"lastBatchElapsedTimeInMilliseconds\":{_lastBatchElapsedTime.TotalMilliseconds}," +
$"\"taskId\":{context.Lease.PartitionId}," +
$"\"batchSize\":{messageCount}," +
$"\"lastBatchFailedDocuments\":{_lastBatchFailedDocuments}," +
$"\"lastBatchAbandonedDocuments\":{_lastBatchAbandonedDocuments}" +
"}";
return(new BulkItem(
indexBaseName: "ingestionstats",
timestamp: now,
enqueueTime: now,
documentType: "batchstats",
documentId: Guid.NewGuid().ToString(),
documentBody: messageBody));
}
private BulkItem GetAbandonDocumentInfo(
DateTime now,
BulkItem abandonedItem,
string errorMessage)
{
var messageBody =
"{" +
$"\"docId\":\"{abandonedItem.DocumentId}\"," +
$"\"docContent\":{JsonConvert.ToString(abandonedItem.DocumentBodyStart)}," +
$"\"lastError\":{JsonConvert.ToString(errorMessage)}," +
$"\"timestamp\":\"{now:o}\"" +
"}";
return(new BulkItem(
indexBaseName: "abandoneddocs",
timestamp: now,
enqueueTime: now,
documentType: "abandoneddocinfo",
documentId: Guid.NewGuid().ToString(),
documentBody: messageBody));
}
private BulkItem GetPerPartitionBatchStats(
PartitionContext context,
DateTime now,
BulkItem lastItem,
BulkItem oldestItemByTimestamp,
BulkItem oldestItemByEnqueueTime,
TimeSpan lag,
TimeSpan maxLag,
TimeSpan lagFromMessageCreation,
TimeSpan maxLagFromMessageCreation)
{
var messageBody =
"{" +
$"\"lastMessageTimestampInBatch\":\"{lastItem.Timestamp:o}\"," +
$"\"lastMessageEnqueueTimeInBatch\":\"{lastItem.EnqueueTime:o}\"," +
$"\"oldestMessageTimestampInBatch\":\"{oldestItemByTimestamp.Timestamp:o}\"," +
$"\"oldestMessageEnqueueTimeInBatch\":\"{oldestItemByEnqueueTime.EnqueueTime:o}\"," +
$"\"lagInMilliseconds\":{lag.TotalMilliseconds}," +
$"\"maxLagInMilliseconds\":{maxLag.TotalMilliseconds}," +
$"\"lagInMinutes\":{lag.TotalMinutes}," +
$"\"maxLagInMinutes\":{maxLag.TotalMinutes}," +
$"\"lagFromMessageCreationTimeInMinutes\":{lagFromMessageCreation.TotalMinutes}," +
$"\"maxLagFromMessageCreationTimeInMinutes\":{maxLagFromMessageCreation.TotalMinutes}," +
$"\"timestamp\":\"{now:o}\"," +
$"\"partitionId\":{context.Lease.PartitionId}," +
$"\"taskId\":{context.Lease.PartitionId}" +
"}";
return(new BulkItem(
indexBaseName: "ingestionstats",
timestamp: now,
enqueueTime: now,
documentType: "perpartitionstats",
documentId: Guid.NewGuid().ToString(),
documentBody: messageBody));
}
/// <summary>
/// Builds the ElasticSearch bulk index request from the given EventHub messages, include ingestionstats data.
/// Outputs a dictionary (<paramref name="items"/>) mapping from document id to the pre-processed BulkItem.
/// Outputs a list (<paramref name="invalidItems"/>) of invalid EventHub messages to be immediately abandoned.
/// </summary>
private string GetBulkBody(PartitionContext context, IEnumerable <EventData> messages, out Dictionary <string, BulkItem> items, out List <BulkItem> invalidItems)
{
var now = DateTime.UtcNow;
var builder = new StringBuilder();
items = new Dictionary <string, BulkItem>();
invalidItems = new List <BulkItem>();
var messageCount = 0;
// Find the last, oldest item by creation time, and oldest item by enqueue time.
BulkItem lastItem = null, oldestItemByTimestamp = null, oldestItemByEnqueueTime = null;
foreach (var message in messages)
{
var bulkItem = BulkItem.FromEventData(message);
// If the item appears to be invalid, mark it to be abandoned immediately.
if (!bulkItem.IsValid)
{
invalidItems.Add(bulkItem);
continue;
}
// Include this item in the bulk index request.
builder.AppendLine(bulkItem.ToBulkString());
items[bulkItem.DocumentId] = bulkItem;
// Remember the last/oldest items.
if (oldestItemByTimestamp == null || bulkItem.Timestamp < oldestItemByTimestamp.Timestamp)
{
oldestItemByTimestamp = bulkItem;
}
if (oldestItemByEnqueueTime == null || bulkItem.EnqueueTime < oldestItemByEnqueueTime.EnqueueTime)
{
oldestItemByEnqueueTime = bulkItem;
}
lastItem = bulkItem;
messageCount++;
}
// Verify we received at least one item.
if (lastItem == null)
{
return(null);
}
if (_sendProcessingStats)
{
// Compute the various lag metrics.
var lag = GetTimeSpanOrZero(now - lastItem.EnqueueTime);
var maxLag = GetTimeSpanOrZero(now - oldestItemByEnqueueTime.EnqueueTime);
var lagFromMessageCreation = GetTimeSpanOrZero(now - lastItem.Timestamp);
var maxLagFromMessageCreation = GetTimeSpanOrZero(now - oldestItemByTimestamp.Timestamp);
// Get the batch statistics.
var batchStats = GetBatchStats(context, now, messageCount, lastItem, oldestItemByTimestamp, oldestItemByEnqueueTime, lag, maxLag, lagFromMessageCreation, maxLagFromMessageCreation);
builder.AppendLine(batchStats.ToBulkString());
items[batchStats.DocumentId] = batchStats;
// Get the partition statistics.
var partitionStats = GetPerPartitionBatchStats(context, now, lastItem, oldestItemByTimestamp, oldestItemByEnqueueTime, lag, maxLag, lagFromMessageCreation, maxLagFromMessageCreation);
builder.AppendLine(partitionStats.ToBulkString());
items[partitionStats.DocumentId] = partitionStats;
}
return(builder.ToString());
}