internal virtual async Task CheckpointAsync(string partitionId, EventData checkpointEvent, CancellationToken cancellationToken = default)
{
string checkpointBlob = string.Format(CultureInfo.InvariantCulture, CheckpointPrefixFormat + partitionId, FullyQualifiedNamespace.ToLowerInvariant(), EventHubName.ToLowerInvariant(), ConsumerGroup.ToLowerInvariant());
Dictionary <string, string> checkpointMetadata = new Dictionary <string, string>()
{
{ OffsetMetadataKey, checkpointEvent.Offset.ToString(CultureInfo.InvariantCulture) },
{ SequenceNumberMetadataKey, checkpointEvent.SequenceNumber.ToString(CultureInfo.InvariantCulture) }
};
using MemoryStream emptyStream = new MemoryStream(Array.Empty <byte>());
await ContainerClient.GetBlobClient(checkpointBlob).UploadAsync(emptyStream, metadata: checkpointMetadata, cancellationToken: cancellationToken).ConfigureAwait(false);
LeaseInfos[partitionId] = new LeaseInfo(checkpointEvent.Offset, checkpointEvent.SequenceNumber);
// In addition to writing a checkpoint in the V5 format, we also write one in the older V4 format, as some processes (e.g. the scale controller) expect
// checkpoints in the older format. This also makes it possible to move to an earlier version of the SDK without having to re-process events.
BlobPartitionLease lease = new BlobPartitionLease()
{
PartitionId = partitionId,
Owner = Identifier,
Offset = checkpointEvent.Offset.ToString(CultureInfo.InvariantCulture),
SequenceNumber = checkpointEvent.SequenceNumber,
};
using MemoryStream legacyCheckpointStream = new MemoryStream();
await JsonSerializer.SerializeAsync(legacyCheckpointStream, lease).ConfigureAwait(false);
string legacyCheckpointBlob = $"{LegacyCheckpointStorageBlobPrefix}{ConsumerGroup}/{partitionId}";
Dictionary <string, string> legacyCheckpoitMetadata = new Dictionary <string, string>()
{
{ OwningHostMedataKey, Identifier }
};
await ContainerClient.GetBlobClient(checkpointBlob).UploadAsync(legacyCheckpointStream, metadata: legacyCheckpoitMetadata, cancellationToken: cancellationToken).ConfigureAwait(false);
}
internal async Task <EventHubsTriggerMetrics> CreateTriggerMetrics(List <EventHubPartitionRuntimeInformation> partitionRuntimeInfo, bool alwaysLog = false)
{
long totalUnprocessedEventCount = 0;
bool logPartitionInfo = alwaysLog ? true : DateTime.UtcNow >= _nextPartitionLogTime;
bool logPartitionWarning = alwaysLog ? true : DateTime.UtcNow >= _nextPartitionWarningTime;
// For each partition, get the last enqueued sequence number.
// If the last enqueued sequence number does not equal the SequenceNumber from the lease info in storage,
// accumulate new event counts across partitions to derive total new event counts.
List <string> partitionErrors = new List <string>();
for (int i = 0; i < partitionRuntimeInfo.Count; i++)
{
long partitionUnprocessedEventCount = 0;
Tuple <BlobPartitionLease, string> partitionLeaseFile = await GetPartitionLeaseFileAsync(i);
BlobPartitionLease partitionLeaseInfo = partitionLeaseFile.Item1;
string errorMsg = partitionLeaseFile.Item2;
if (partitionRuntimeInfo[i] == null || partitionLeaseInfo == null)
{
partitionErrors.Add(errorMsg);
}
else
{
// Check for the unprocessed messages when there are messages on the event hub parition
// In that case, LastEnqueuedSequenceNumber will be >= 0
if ((partitionRuntimeInfo[i].LastEnqueuedSequenceNumber != -1 && partitionRuntimeInfo[i].LastEnqueuedSequenceNumber != partitionLeaseInfo.SequenceNumber) ||
(partitionLeaseInfo.Offset == null && partitionRuntimeInfo[i].LastEnqueuedSequenceNumber >= 0))
{
partitionUnprocessedEventCount = GetUnprocessedEventCount(partitionRuntimeInfo[i], partitionLeaseInfo);
totalUnprocessedEventCount += partitionUnprocessedEventCount;
}
}
}
// Only log if not all partitions are failing or it's time to log
if (partitionErrors.Count > 0 && (partitionErrors.Count != partitionRuntimeInfo.Count || logPartitionWarning))
{
_logger.LogWarning($"Function '{_functionId}': Unable to deserialize partition or lease info with the " +
$"following errors: {string.Join(" ", partitionErrors)}");
_nextPartitionWarningTime = DateTime.UtcNow.AddMinutes(PartitionLogIntervalInMinutes);
}
if (totalUnprocessedEventCount > 0 && logPartitionInfo)
{
_logger.LogInformation($"Function '{_functionId}', Total new events: {totalUnprocessedEventCount}");
_nextPartitionLogTime = DateTime.UtcNow.AddMinutes(PartitionLogIntervalInMinutes);
}
return(new EventHubsTriggerMetrics
{
Timestamp = DateTime.UtcNow,
PartitionCount = partitionRuntimeInfo.Count,
EventCount = totalUnprocessedEventCount
});
}
protected override async Task <IEnumerable <EventProcessorCheckpoint> > ListCheckpointsAsync(CancellationToken cancellationToken)
{
// First, we read information from the location that the EventHubs V5 SDK writes to.
Dictionary <string, EventProcessorCheckpoint> checkpoints = new Dictionary <string, EventProcessorCheckpoint>();
string checkpointBlobsPrefix = string.Format(CultureInfo.InvariantCulture, CheckpointPrefixFormat, FullyQualifiedNamespace.ToLowerInvariant(), EventHubName.ToLowerInvariant(), ConsumerGroup.ToLowerInvariant());
await foreach (BlobItem item in ContainerClient.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: checkpointBlobsPrefix, cancellationToken: cancellationToken).ConfigureAwait(false))
{
if (long.TryParse(item.Metadata[OffsetMetadataKey], NumberStyles.Integer, CultureInfo.InvariantCulture, out long offset) &&
long.TryParse(item.Metadata[SequenceNumberMetadataKey], NumberStyles.Integer, CultureInfo.InvariantCulture, out long sequenceNumber))
{
string partitionId = item.Name.Substring(checkpointBlobsPrefix.Length);
LeaseInfos.TryAdd(partitionId, new LeaseInfo(offset, sequenceNumber));
checkpoints.Add(partitionId, new EventProcessorCheckpoint()
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
PartitionId = partitionId,
StartingPosition = EventPosition.FromOffset(offset, isInclusive: false)
});
}
}
// Check to see if there are any additional checkpoints in the older location that the V4 SDK would write to. If so, use them (this is helpful when moving from the V4 to V5 SDK,
// since it means we will not have to reprocess messages processed and checkpointed by the older SDK).
string legacyCheckpointAndOwnershipPrefix = $"{LegacyCheckpointStorageBlobPrefix}{ConsumerGroup}/";
await foreach (BlobItem item in ContainerClient.GetBlobsAsync(prefix: legacyCheckpointAndOwnershipPrefix, cancellationToken: cancellationToken).ConfigureAwait(false))
{
string partitionId = item.Name.Substring(legacyCheckpointAndOwnershipPrefix.Length);
if (!checkpoints.ContainsKey(partitionId))
{
using MemoryStream checkpointStream = new MemoryStream();
await ContainerClient.GetBlobClient(item.Name).DownloadToAsync(checkpointStream, cancellationToken: cancellationToken).ConfigureAwait(false);
checkpointStream.Position = 0;
BlobPartitionLease lease = await JsonSerializer.DeserializeAsync <BlobPartitionLease>(checkpointStream, cancellationToken : cancellationToken).ConfigureAwait(false);
if (long.TryParse(lease.Offset, out long offset))
{
LeaseInfos.TryAdd(partitionId, new LeaseInfo(offset, lease.SequenceNumber ?? 0));
checkpoints.Add(partitionId, new EventProcessorCheckpoint()
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
PartitionId = partitionId,
StartingPosition = EventPosition.FromOffset(offset, isInclusive: false)
});
}
}
}
return(checkpoints.Values);
}
private async Task <Tuple <BlobPartitionLease, string> > GetPartitionLeaseFileAsync(int partitionId)
{
BlobPartitionLease blobPartitionLease = null;
string prefix = $"{EventHubOptions.GetBlobPrefix(_eventHubName, EventHubOptions.GetEventHubNamespace(ConnectionStringBuilder))}{_consumerGroup}/{partitionId}";
string errorMsg = null;
try
{
CloudBlockBlob blockBlob = BlobContainer.GetBlockBlobReference(prefix);
if (blockBlob != null)
{
var result = await blockBlob.DownloadTextAsync();
if (!string.IsNullOrEmpty(result))
{
blobPartitionLease = JsonConvert.DeserializeObject <BlobPartitionLease>(result);
}
}
}
catch (Exception e)
{
var storageException = e as StorageException;
if (storageException?.RequestInformation?.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
errorMsg = $"Lease file data could not be found for blob on Partition: '{partitionId}', " +
$"EventHub: '{_eventHubName}', '{_consumerGroup}'. Error: {e.Message}";
}
else if (e is JsonSerializationException)
{
errorMsg = $"Could not deserialize blob lease info for blob on Partition: '{partitionId}', " +
$"EventHub: '{_eventHubName}', Consumer Group: '{_consumerGroup}'. Error: {e.Message}";
}
else
{
errorMsg = $"Encountered exception while checking for last checkpointed sequence number for blob " +
$"on Partition: '{partitionId}', EventHub: '{_eventHubName}', Consumer Group: '{_consumerGroup}'. Error: {e.Message}";
}
}
return(new Tuple <BlobPartitionLease, string>(blobPartitionLease, errorMsg));
}
// Get the number of unprocessed events by deriving the delta between the server side info and the partition lease info,
private long GetUnprocessedEventCount(EventHubPartitionRuntimeInformation partitionInfo, BlobPartitionLease partitionLeaseInfo)
{
long partitionLeaseInfoSequenceNumber = partitionLeaseInfo.SequenceNumber ?? 0;
// This handles two scenarios:
// 1. If the partition has received its first message, Offset will be null and LastEnqueuedSequenceNumber will be 0
// 2. If there are no instances set to process messages, Offset will be null and LastEnqueuedSequenceNumber will be >= 0
if (partitionLeaseInfo.Offset == null && partitionInfo.LastEnqueuedSequenceNumber >= 0)
{
return(partitionInfo.LastEnqueuedSequenceNumber + 1);
}
if (partitionInfo.LastEnqueuedSequenceNumber > partitionLeaseInfoSequenceNumber)
{
return(partitionInfo.LastEnqueuedSequenceNumber - partitionLeaseInfoSequenceNumber);
}
// Partition is a circular buffer, so it is possible that
// LastEnqueuedSequenceNumber < SequenceNumber
long count = 0;
unchecked
{
count = (long.MaxValue - partitionInfo.LastEnqueuedSequenceNumber) + partitionLeaseInfoSequenceNumber;
}
// It's possible for checkpointing to be ahead of the partition's LastEnqueuedSequenceNumber,
// especially if checkpointing is happening often and load is very low.
// If count is negative, we need to know that this read is invalid, so return 0.
// e.g., (9223372036854775807 - 10) + 11 = -9223372036854775808
return((count < 0) ? 0 : count);
}
