// Allow subclasses to call CheckpointAsync to store checkpoint information without
// having to understand the details of how checkpoints are stored.
protected async Task CheckpointAsync(
EventProcessorPartition partition,
EventData data,
CancellationToken cancellationToken = default)
{
using MemoryStream emptyStream = new MemoryStream(Array.Empty <byte>());
string checkpointBlob = string.Format(
CheckpointPrefixFormat + partition.PartitionId,
FullyQualifiedNamespace.ToLowerInvariant(),
EventHubName.ToLowerInvariant(),
ConsumerGroup.ToLowerInvariant());
Dictionary <string, string> checkpointMetadata = new Dictionary <string, string>()
{
{ OffsetMetadataKey, data.Offset.ToString(CultureInfo.InvariantCulture) },
};
await StorageContainer
.GetBlobClient(checkpointBlob)
.UploadAsync(
emptyStream,
metadata: checkpointMetadata,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
}
// Ownership information is stored as metadata on blobs in Azure Storage. To list ownership
// information we list all the blobs in Blob Storage (with their corresponding metadata) and
// then extract the ownership information from the metadata.
protected override async Task <IEnumerable <EventProcessorPartitionOwnership> > ListOwnershipAsync(
CancellationToken cancellationToken = default)
{
List <EventProcessorPartitionOwnership> partitonOwnerships =
new List <EventProcessorPartitionOwnership>();
string ownershipBlobsPefix = string.Format(
OwnershipPrefixFormat,
FullyQualifiedNamespace.ToLowerInvariant(),
EventHubName.ToLowerInvariant(),
ConsumerGroup.ToLowerInvariant());
AsyncPageable <BlobItem> blobItems = StorageContainer.GetBlobsAsync(
traits: BlobTraits.Metadata,
prefix: ownershipBlobsPefix,
cancellationToken: cancellationToken);
await foreach (BlobItem blob in blobItems.ConfigureAwait(false))
{
partitonOwnerships.Add(new EventProcessorPartitionOwnership
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
LastModifiedTime = blob.Properties.LastModified.GetValueOrDefault(),
OwnerIdentifier = blob.Metadata[OwnerIdentifierMetadataKey],
PartitionId = blob.Name.Substring(ownershipBlobsPefix.Length),
Version = blob.Properties.ETag.ToString()
});
}
return(partitonOwnerships);
}
protected override void InitializeTarget()
{
base.InitializeTarget();
var entityPath = EventHubName?.Render(LogEventInfo.CreateNullEvent())?.Trim() ?? string.Empty;
var connectionString = ConnectionString?.Render(LogEventInfo.CreateNullEvent()) ?? string.Empty;
if (string.IsNullOrWhiteSpace(connectionString))
{
throw new ArgumentException("ConnectionString is required");
}
_eventHubService.Connect(connectionString, entityPath);
}
// We use the same strategy for recording checkpoint information as ownership information
// (metadata on a blob in blob storage)
protected override async Task <EventProcessorCheckpoint> GetCheckpointAsync(
string partitionId,
CancellationToken cancellationToken = default)
{
try
{
string blobName = string.Format(
CheckpointPrefixFormat + partitionId,
FullyQualifiedNamespace.ToLowerInvariant(),
EventHubName.ToLowerInvariant(),
ConsumerGroup.ToLowerInvariant());
Response <BlobProperties> blobResponse = await StorageContainer
.GetBlobClient(blobName)
.GetPropertiesAsync(cancellationToken: cancellationToken)
.ConfigureAwait(false);
if (long.TryParse(
blobResponse.Value.Metadata[OffsetMetadataKey],
NumberStyles.Integer,
CultureInfo.InvariantCulture,
out long offset))
{
return(new EventProcessorCheckpoint
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
PartitionId = partitionId,
StartingPosition = EventPosition.FromOffset(offset, isInclusive: false)
});
}
}
catch (RequestFailedException ex) when(ex.Status == 404)
{
// Ignore; this will occur when no checkpoint is available.
}
// Returning null will signal that the default starting position
// should be used for this partition.
return(null);
}
internal virtual async Task CheckpointAsync(string partitionId, EventData checkpointEvent, CancellationToken cancellationToken = default)
{
string checkpointBlob = string.Format(CultureInfo.InvariantCulture, CheckpointPrefixFormat + partitionId, FullyQualifiedNamespace.ToLowerInvariant(), EventHubName.ToLowerInvariant(), ConsumerGroup.ToLowerInvariant());
Dictionary <string, string> checkpointMetadata = new Dictionary <string, string>()
{
{ OffsetMetadataKey, checkpointEvent.Offset.ToString(CultureInfo.InvariantCulture) },
{ SequenceNumberMetadataKey, checkpointEvent.SequenceNumber.ToString(CultureInfo.InvariantCulture) }
};
using MemoryStream emptyStream = new MemoryStream(Array.Empty <byte>());
await ContainerClient.GetBlobClient(checkpointBlob).UploadAsync(emptyStream, metadata: checkpointMetadata, cancellationToken: cancellationToken).ConfigureAwait(false);
LeaseInfos[partitionId] = new LeaseInfo(checkpointEvent.Offset, checkpointEvent.SequenceNumber);
// In addition to writing a checkpoint in the V5 format, we also write one in the older V4 format, as some processes (e.g. the scale controller) expect
// checkpoints in the older format. This also makes it possible to move to an earlier version of the SDK without having to re-process events.
BlobPartitionLease lease = new BlobPartitionLease()
{
PartitionId = partitionId,
Owner = Identifier,
Offset = checkpointEvent.Offset.ToString(CultureInfo.InvariantCulture),
SequenceNumber = checkpointEvent.SequenceNumber,
};
using MemoryStream legacyCheckpointStream = new MemoryStream();
await JsonSerializer.SerializeAsync(legacyCheckpointStream, lease).ConfigureAwait(false);
string legacyCheckpointBlob = $"{LegacyCheckpointStorageBlobPrefix}{ConsumerGroup}/{partitionId}";
Dictionary <string, string> legacyCheckpoitMetadata = new Dictionary <string, string>()
{
{ OwningHostMedataKey, Identifier }
};
await ContainerClient.GetBlobClient(checkpointBlob).UploadAsync(legacyCheckpointStream, metadata: legacyCheckpoitMetadata, cancellationToken: cancellationToken).ConfigureAwait(false);
}
protected override async Task <IEnumerable <EventProcessorCheckpoint> > ListCheckpointsAsync(CancellationToken cancellationToken)
{
// First, we read information from the location that the EventHubs V5 SDK writes to.
Dictionary <string, EventProcessorCheckpoint> checkpoints = new Dictionary <string, EventProcessorCheckpoint>();
string checkpointBlobsPrefix = string.Format(CultureInfo.InvariantCulture, CheckpointPrefixFormat, FullyQualifiedNamespace.ToLowerInvariant(), EventHubName.ToLowerInvariant(), ConsumerGroup.ToLowerInvariant());
await foreach (BlobItem item in ContainerClient.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: checkpointBlobsPrefix, cancellationToken: cancellationToken).ConfigureAwait(false))
{
if (long.TryParse(item.Metadata[OffsetMetadataKey], NumberStyles.Integer, CultureInfo.InvariantCulture, out long offset) &&
long.TryParse(item.Metadata[SequenceNumberMetadataKey], NumberStyles.Integer, CultureInfo.InvariantCulture, out long sequenceNumber))
{
string partitionId = item.Name.Substring(checkpointBlobsPrefix.Length);
LeaseInfos.TryAdd(partitionId, new LeaseInfo(offset, sequenceNumber));
checkpoints.Add(partitionId, new EventProcessorCheckpoint()
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
PartitionId = partitionId,
StartingPosition = EventPosition.FromOffset(offset, isInclusive: false)
});
}
}
// Check to see if there are any additional checkpoints in the older location that the V4 SDK would write to. If so, use them (this is helpful when moving from the V4 to V5 SDK,
// since it means we will not have to reprocess messages processed and checkpointed by the older SDK).
string legacyCheckpointAndOwnershipPrefix = $"{LegacyCheckpointStorageBlobPrefix}{ConsumerGroup}/";
await foreach (BlobItem item in ContainerClient.GetBlobsAsync(prefix: legacyCheckpointAndOwnershipPrefix, cancellationToken: cancellationToken).ConfigureAwait(false))
{
string partitionId = item.Name.Substring(legacyCheckpointAndOwnershipPrefix.Length);
if (!checkpoints.ContainsKey(partitionId))
{
using MemoryStream checkpointStream = new MemoryStream();
await ContainerClient.GetBlobClient(item.Name).DownloadToAsync(checkpointStream, cancellationToken: cancellationToken).ConfigureAwait(false);
checkpointStream.Position = 0;
BlobPartitionLease lease = await JsonSerializer.DeserializeAsync <BlobPartitionLease>(checkpointStream, cancellationToken : cancellationToken).ConfigureAwait(false);
if (long.TryParse(lease.Offset, out long offset))
{
LeaseInfos.TryAdd(partitionId, new LeaseInfo(offset, lease.SequenceNumber ?? 0));
checkpoints.Add(partitionId, new EventProcessorCheckpoint()
{
ConsumerGroup = ConsumerGroup,
EventHubName = EventHubName,
FullyQualifiedNamespace = FullyQualifiedNamespace,
PartitionId = partitionId,
StartingPosition = EventPosition.FromOffset(offset, isInclusive: false)
});
}
}
}
return(checkpoints.Values);
}
