/// <summary>
/// Creates a checkpoint instance based on the blob name for a legacy checkpoint format.
/// </summary>
///
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace the checkpoint are associated with. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param>
/// <param name="eventHubName">The name of the specific Event Hub the checkpoint is associated with, relative to the Event Hubs namespace that contains it.</param>
/// <param name="consumerGroup">The name of the consumer group the checkpoint is associated with.</param>
/// <param name="partitionId">The partition id the specific checkpoint is associated with.</param>
/// <param name="blobName">The name of the blob that represents the checkpoint.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param>
///
/// <returns>A <see cref="EventProcessorCheckpoint"/> initialized with checkpoint properties if the checkpoint exists, otherwise <code>null</code>.</returns>
///
private async Task <EventProcessorCheckpoint> CreateLegacyCheckpoint(string fullyQualifiedNamespace,
string eventHubName,
string consumerGroup,
string blobName,
string partitionId,
CancellationToken cancellationToken)
{
var startingPosition = default(EventPosition?);
BlobBaseClient blobClient = ContainerClient.GetBlobClient(blobName);
using var memoryStream = new MemoryStream();
await blobClient.DownloadToAsync(memoryStream, cancellationToken).ConfigureAwait(false);
if (TryReadLegacyCheckpoint(
memoryStream.GetBuffer().AsSpan(0, (int)memoryStream.Length),
out long?offset,
out long?sequenceNumber))
{
if (offset.HasValue)
{
startingPosition = EventPosition.FromOffset(offset.Value, false);
}
else
{
// Skip checkpoints without an offset without logging an error.
return(null);
}
}
if (!startingPosition.HasValue)
{
InvalidCheckpointFound(partitionId, fullyQualifiedNamespace, eventHubName, consumerGroup);
return(null);
}
return(new BlobStorageCheckpoint
{
FullyQualifiedNamespace = fullyQualifiedNamespace,
EventHubName = eventHubName,
ConsumerGroup = consumerGroup,
PartitionId = partitionId,
StartingPosition = startingPosition.Value,
Offset = offset,
SequenceNumber = sequenceNumber
});
}
/// <summary>
/// Download blob to local file
/// </summary>
/// <param name="blob">Source blob object</param>
/// <param name="filePath">Destination file path</param>
internal virtual async Task DownloadBlob(long taskId, IStorageBlobManagement localChannel, BlobBaseClient blob, string filePath)
{
Track2Models.BlobProperties blobProperties = blob.GetProperties(cancellationToken: CmdletCancellationToken);
if (this.Force.IsPresent ||
!System.IO.File.Exists(filePath) ||
ShouldContinue(string.Format(Resources.OverwriteConfirmation, filePath), null))
{
StorageTransferOptions trasnferOption = new StorageTransferOptions()
{
MaximumConcurrency = this.GetCmdletConcurrency(),
MaximumTransferSize = size4MB,
InitialTransferSize = size4MB
};
await blob.DownloadToAsync(filePath, BlobRequestConditions, trasnferOption, CmdletCancellationToken).ConfigureAwait(false);
OutputStream.WriteObject(taskId, new AzureStorageBlob(blob, localChannel.StorageContext, blobProperties, options: ClientOptions));
}
}
/// <summary>
/// Retrieves a list of all the checkpoints in a data store for a given namespace, Event Hub and consumer group.
/// </summary>
///
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace the ownership are associated with. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param>
/// <param name="eventHubName">The name of the specific Event Hub the ownership are associated with, relative to the Event Hubs namespace that contains it.</param>
/// <param name="consumerGroup">The name of the consumer group the ownership are associated with.</param>
/// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param>
///
/// <returns>An enumerable containing all the existing checkpoints for the associated Event Hub and consumer group.</returns>
///
public override async Task <IEnumerable <EventProcessorCheckpoint> > ListCheckpointsAsync(string fullyQualifiedNamespace,
string eventHubName,
string consumerGroup,
CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>();
ListCheckpointsStart(fullyQualifiedNamespace, eventHubName, consumerGroup);
async Task <List <EventProcessorCheckpoint> > listCheckpointsAsync(CancellationToken listCheckpointsToken)
{
var prefix = string.Format(CultureInfo.InvariantCulture, CheckpointPrefix, fullyQualifiedNamespace.ToLowerInvariant(), eventHubName.ToLowerInvariant(), consumerGroup.ToLowerInvariant());
var checkpoints = new List <EventProcessorCheckpoint>();
await foreach (BlobItem blob in ContainerClient.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: prefix, cancellationToken: listCheckpointsToken).ConfigureAwait(false))
{
var partitionId = blob.Name.Substring(prefix.Length);
var startingPosition = default(EventPosition?);
var offset = default(long?);
var sequenceNumber = default(long?);
if (blob.Metadata.TryGetValue(BlobMetadataKey.Offset, out var str) && long.TryParse(str, NumberStyles.Integer, CultureInfo.InvariantCulture, out var result))
{
offset = result;
startingPosition = EventPosition.FromOffset(result, false);
}
else if (blob.Metadata.TryGetValue(BlobMetadataKey.SequenceNumber, out str) && long.TryParse(str, NumberStyles.Integer, CultureInfo.InvariantCulture, out result))
{
sequenceNumber = result;
startingPosition = EventPosition.FromSequenceNumber(result, false);
}
// If either the offset or the sequence number was not populated,
// this is not a valid checkpoint.
if (startingPosition.HasValue)
{
checkpoints.Add(new BlobStorageCheckpoint
{
FullyQualifiedNamespace = fullyQualifiedNamespace,
EventHubName = eventHubName,
ConsumerGroup = consumerGroup,
PartitionId = partitionId,
StartingPosition = startingPosition.Value,
Offset = offset,
SequenceNumber = sequenceNumber
});
}
else
{
InvalidCheckpointFound(partitionId, fullyQualifiedNamespace, eventHubName, consumerGroup);
}
}
return(checkpoints);
};
async Task <List <EventProcessorCheckpoint> > listLegacyCheckpointsAsync(List <EventProcessorCheckpoint> existingCheckpoints, CancellationToken listCheckpointsToken)
{
// Legacy checkpoints are not normalized to lowercase
var legacyPrefix = string.Format(CultureInfo.InvariantCulture, LegacyCheckpointPrefix, fullyQualifiedNamespace, eventHubName, consumerGroup);
var checkpoints = new List <EventProcessorCheckpoint>();
await foreach (BlobItem blob in ContainerClient.GetBlobsAsync(prefix: legacyPrefix, cancellationToken: listCheckpointsToken).ConfigureAwait(false))
{
// Skip new checkpoints and empty blobs
if (blob.Properties.ContentLength == 0)
{
continue;
}
var partitionId = blob.Name.Substring(legacyPrefix.Length);
// Check whether there is already a checkpoint for this partition id
if (existingCheckpoints.Any(existingCheckpoint => string.Equals(existingCheckpoint.PartitionId, partitionId, StringComparison.Ordinal)))
{
continue;
}
var startingPosition = default(EventPosition?);
BlobBaseClient blobClient = ContainerClient.GetBlobClient(blob.Name);
using var memoryStream = new MemoryStream();
await blobClient.DownloadToAsync(memoryStream, listCheckpointsToken).ConfigureAwait(false);
TryReadLegacyCheckpoint(
memoryStream.GetBuffer().AsSpan(0, (int)memoryStream.Length),
out long?offset,
out long?sequenceNumber);
if (offset.HasValue)
{
startingPosition = EventPosition.FromOffset(offset.Value, false);
}
else if (sequenceNumber.HasValue)
{
startingPosition = EventPosition.FromSequenceNumber(sequenceNumber.Value, false);
}
if (startingPosition.HasValue)
{
checkpoints.Add(new BlobStorageCheckpoint
{
FullyQualifiedNamespace = fullyQualifiedNamespace,
EventHubName = eventHubName,
ConsumerGroup = consumerGroup,
PartitionId = partitionId,
StartingPosition = startingPosition.Value,
Offset = offset,
SequenceNumber = sequenceNumber
});
}
else
{
InvalidCheckpointFound(partitionId, fullyQualifiedNamespace, eventHubName, consumerGroup);
}
}
return(checkpoints);
};
List <EventProcessorCheckpoint> checkpoints = null;
try
{
checkpoints = await ApplyRetryPolicy(listCheckpointsAsync, cancellationToken).ConfigureAwait(false);
if (InitializeWithLegacyCheckpoints)
{
checkpoints.AddRange(await ApplyRetryPolicy(ct => listLegacyCheckpointsAsync(checkpoints, ct), cancellationToken).ConfigureAwait(false));
}
return(checkpoints);
}
catch (RequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.ContainerNotFound)
{
ListCheckpointsError(fullyQualifiedNamespace, eventHubName, consumerGroup, ex);
throw new RequestFailedException(BlobsResourceDoesNotExist);
}
catch (Exception ex)
{
ListCheckpointsError(fullyQualifiedNamespace, eventHubName, consumerGroup, ex);
throw;
}
finally
{
ListCheckpointsComplete(fullyQualifiedNamespace, eventHubName, consumerGroup, checkpoints?.Count ?? 0);
}
}
