internal async Task <EventHubsTriggerMetrics> CreateTriggerMetrics(List <PartitionProperties> partitionRuntimeInfo, bool alwaysLog = false)
{
long totalUnprocessedEventCount = 0;
bool logPartitionInfo = alwaysLog ? true : DateTime.UtcNow >= _nextPartitionLogTime;
bool logPartitionWarning = alwaysLog ? true : DateTime.UtcNow >= _nextPartitionWarningTime;
// For each partition, get the last enqueued sequence number.
// If the last enqueued sequence number does not equal the SequenceNumber from the lease info in storage,
// accumulate new event counts across partitions to derive total new event counts.
List <string> partitionErrors = new List <string>();
for (int i = 0; i < partitionRuntimeInfo.Count; i++)
{
Tuple <BlobParitionCheckpoint, string> partitionLeaseFile = await GetPartitionLeaseFileAsync(i).ConfigureAwait(false);
BlobParitionCheckpoint partitionLeaseInfo = partitionLeaseFile.Item1;
string errorMsg = partitionLeaseFile.Item2;
if (partitionRuntimeInfo[i] == null || partitionLeaseInfo == null)
{
partitionErrors.Add(errorMsg);
}
else
{
// Check for the unprocessed messages when there are messages on the event hub parition
// In that case, LastEnqueuedSequenceNumber will be >= 0
if ((partitionRuntimeInfo[i].LastEnqueuedSequenceNumber != -1 && partitionRuntimeInfo[i].LastEnqueuedSequenceNumber != partitionLeaseInfo.SequenceNumber) ||
(partitionLeaseInfo.Offset == null && partitionRuntimeInfo[i].LastEnqueuedSequenceNumber >= 0))
{
long partitionUnprocessedEventCount = GetUnprocessedEventCount(partitionRuntimeInfo[i], partitionLeaseInfo);
totalUnprocessedEventCount += partitionUnprocessedEventCount;
}
}
}
// Only log if not all partitions are failing or it's time to log
if (partitionErrors.Count > 0 && (partitionErrors.Count != partitionRuntimeInfo.Count || logPartitionWarning))
{
_logger.LogWarning($"Function '{_functionId}': Unable to deserialize partition or lease info with the " +
$"following errors: {string.Join(" ", partitionErrors)}");
_nextPartitionWarningTime = DateTime.UtcNow.AddMinutes(PartitionLogIntervalInMinutes);
}
if (totalUnprocessedEventCount > 0 && logPartitionInfo)
{
_logger.LogInformation($"Function '{_functionId}', Total new events: {totalUnprocessedEventCount}");
_nextPartitionLogTime = DateTime.UtcNow.AddMinutes(PartitionLogIntervalInMinutes);
}
return(new EventHubsTriggerMetrics
{
Timestamp = DateTime.UtcNow,
PartitionCount = partitionRuntimeInfo.Count,
EventCount = totalUnprocessedEventCount
});
}
private async Task <Tuple <BlobParitionCheckpoint, string> > GetPartitionLeaseFileAsync(int partitionId)
{
BlobParitionCheckpoint blobParitionCheckpoint = null;
string prefix = $"{EventHubOptions.GetBlobPrefix(_eventHubName, _client.Value.FullyQualifiedNamespace)}{_consumerGroup}/checkpoint/{partitionId}";
string errorMsg = null;
try
{
BlobClient blockBlob = BlobContainer.GetBlobClient(prefix);
BlobProperties properties = await blockBlob.GetPropertiesAsync().ConfigureAwait(false);
if (properties.Metadata.TryGetValue(SequenceNumberMetadataName, out string sequenceNumberString))
{
blobParitionCheckpoint ??= new BlobParitionCheckpoint();
blobParitionCheckpoint.SequenceNumber = long.Parse(sequenceNumberString, CultureInfo.InvariantCulture);
}
if (properties.Metadata.TryGetValue(OffsetMetadataName, out string offsetString))
{
blobParitionCheckpoint ??= new BlobParitionCheckpoint();
blobParitionCheckpoint.Offset = long.Parse(offsetString, CultureInfo.InvariantCulture);
}
if (blobParitionCheckpoint == null)
{
errorMsg = $"Checkpoint file did not contain required metadata on Partition: '{partitionId}', " +
$"EventHub: '{_eventHubName}', '{_consumerGroup}'.";
}
}
catch (RequestFailedException e) when(e.Status == (int)HttpStatusCode.NotFound)
{
errorMsg = $"Checkpoint file data could not be found for blob on Partition: '{partitionId}', " +
$"EventHub: '{_eventHubName}', '{_consumerGroup}'. Error: {e.Message}";
}
catch (Exception e)
{
errorMsg = $"Encountered exception while checking for last checkpointed sequence number for blob " +
$"on Partition: '{partitionId}', EventHub: '{_eventHubName}', Consumer Group: '{_consumerGroup}'. Error: {e.Message}";
}
return(new Tuple <BlobParitionCheckpoint, string>(blobParitionCheckpoint, errorMsg));
}
// Get the number of unprocessed events by deriving the delta between the server side info and the partition lease info,
private static long GetUnprocessedEventCount(PartitionProperties partitionInfo, BlobParitionCheckpoint partitionLeaseInfo)
{
long partitionLeaseInfoSequenceNumber = partitionLeaseInfo.SequenceNumber ?? 0;
// This handles two scenarios:
// 1. If the partition has received its first message, Offset will be null and LastEnqueuedSequenceNumber will be 0
// 2. If there are no instances set to process messages, Offset will be null and LastEnqueuedSequenceNumber will be >= 0
if (partitionLeaseInfo.Offset == null && partitionInfo.LastEnqueuedSequenceNumber >= 0)
{
return(partitionInfo.LastEnqueuedSequenceNumber + 1);
}
if (partitionInfo.LastEnqueuedSequenceNumber > partitionLeaseInfoSequenceNumber)
{
return(partitionInfo.LastEnqueuedSequenceNumber - partitionLeaseInfoSequenceNumber);
}
// Partition is a circular buffer, so it is possible that
// LastEnqueuedSequenceNumber < SequenceNumber
long count = 0;
unchecked
{
count = (long.MaxValue - partitionInfo.LastEnqueuedSequenceNumber) + partitionLeaseInfoSequenceNumber;
}
// It's possible for checkpointing to be ahead of the partition's LastEnqueuedSequenceNumber,
// especially if checkpointing is happening often and load is very low.
// If count is negative, we need to know that this read is invalid, so return 0.
// e.g., (9223372036854775807 - 10) + 11 = -9223372036854775808
return((count < 0) ? 0 : count);
}
