/// <summary> /// Creates a checkpoint instance based on the blob name for a legacy checkpoint format. /// </summary> /// /// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace the checkpoint are associated with. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param> /// <param name="eventHubName">The name of the specific Event Hub the checkpoint is associated with, relative to the Event Hubs namespace that contains it.</param> /// <param name="consumerGroup">The name of the consumer group the checkpoint is associated with.</param> /// <param name="partitionId">The partition id the specific checkpoint is associated with.</param> /// <param name="blobName">The name of the blob that represents the checkpoint.</param> /// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param> /// /// <returns>A <see cref="EventProcessorCheckpoint"/> initialized with checkpoint properties if the checkpoint exists, otherwise <code>null</code>.</returns> /// private async Task <EventProcessorCheckpoint> CreateLegacyCheckpoint(string fullyQualifiedNamespace, string eventHubName, string consumerGroup, string blobName, string partitionId, CancellationToken cancellationToken) { var startingPosition = default(EventPosition?); BlobBaseClient blobClient = ContainerClient.GetBlobClient(blobName); using var memoryStream = new MemoryStream(); await blobClient.DownloadToAsync(memoryStream, cancellationToken).ConfigureAwait(false); if (TryReadLegacyCheckpoint( memoryStream.GetBuffer().AsSpan(0, (int)memoryStream.Length), out long?offset, out long?sequenceNumber)) { if (offset.HasValue) { startingPosition = EventPosition.FromOffset(offset.Value, false); } else { // Skip checkpoints without an offset without logging an error. return(null); } } if (!startingPosition.HasValue) { InvalidCheckpointFound(partitionId, fullyQualifiedNamespace, eventHubName, consumerGroup); return(null); } return(new BlobStorageCheckpoint { FullyQualifiedNamespace = fullyQualifiedNamespace, EventHubName = eventHubName, ConsumerGroup = consumerGroup, PartitionId = partitionId, StartingPosition = startingPosition.Value, Offset = offset, SequenceNumber = sequenceNumber }); }
/// <summary> /// Download blob to local file /// </summary> /// <param name="blob">Source blob object</param> /// <param name="filePath">Destination file path</param> internal virtual async Task DownloadBlob(long taskId, IStorageBlobManagement localChannel, BlobBaseClient blob, string filePath) { Track2Models.BlobProperties blobProperties = blob.GetProperties(cancellationToken: CmdletCancellationToken); if (this.Force.IsPresent || !System.IO.File.Exists(filePath) || ShouldContinue(string.Format(Resources.OverwriteConfirmation, filePath), null)) { StorageTransferOptions trasnferOption = new StorageTransferOptions() { MaximumConcurrency = this.GetCmdletConcurrency(), MaximumTransferSize = size4MB, InitialTransferSize = size4MB }; await blob.DownloadToAsync(filePath, BlobRequestConditions, trasnferOption, CmdletCancellationToken).ConfigureAwait(false); OutputStream.WriteObject(taskId, new AzureStorageBlob(blob, localChannel.StorageContext, blobProperties, options: ClientOptions)); } }
/// <summary> /// Retrieves a list of all the checkpoints in a data store for a given namespace, Event Hub and consumer group. /// </summary> /// /// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace the ownership are associated with. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param> /// <param name="eventHubName">The name of the specific Event Hub the ownership are associated with, relative to the Event Hubs namespace that contains it.</param> /// <param name="consumerGroup">The name of the consumer group the ownership are associated with.</param> /// <param name="cancellationToken">A <see cref="CancellationToken" /> instance to signal the request to cancel the operation.</param> /// /// <returns>An enumerable containing all the existing checkpoints for the associated Event Hub and consumer group.</returns> /// public override async Task <IEnumerable <EventProcessorCheckpoint> > ListCheckpointsAsync(string fullyQualifiedNamespace, string eventHubName, string consumerGroup, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested <TaskCanceledException>(); ListCheckpointsStart(fullyQualifiedNamespace, eventHubName, consumerGroup); async Task <List <EventProcessorCheckpoint> > listCheckpointsAsync(CancellationToken listCheckpointsToken) { var prefix = string.Format(CultureInfo.InvariantCulture, CheckpointPrefix, fullyQualifiedNamespace.ToLowerInvariant(), eventHubName.ToLowerInvariant(), consumerGroup.ToLowerInvariant()); var checkpoints = new List <EventProcessorCheckpoint>(); await foreach (BlobItem blob in ContainerClient.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: prefix, cancellationToken: listCheckpointsToken).ConfigureAwait(false)) { var partitionId = blob.Name.Substring(prefix.Length); var startingPosition = default(EventPosition?); var offset = default(long?); var sequenceNumber = default(long?); if (blob.Metadata.TryGetValue(BlobMetadataKey.Offset, out var str) && long.TryParse(str, NumberStyles.Integer, CultureInfo.InvariantCulture, out var result)) { offset = result; startingPosition = EventPosition.FromOffset(result, false); } else if (blob.Metadata.TryGetValue(BlobMetadataKey.SequenceNumber, out str) && long.TryParse(str, NumberStyles.Integer, CultureInfo.InvariantCulture, out result)) { sequenceNumber = result; startingPosition = EventPosition.FromSequenceNumber(result, false); } // If either the offset or the sequence number was not populated, // this is not a valid checkpoint. if (startingPosition.HasValue) { checkpoints.Add(new BlobStorageCheckpoint { FullyQualifiedNamespace = fullyQualifiedNamespace, EventHubName = eventHubName, ConsumerGroup = consumerGroup, PartitionId = partitionId, StartingPosition = startingPosition.Value, Offset = offset, SequenceNumber = sequenceNumber }); } else { InvalidCheckpointFound(partitionId, fullyQualifiedNamespace, eventHubName, consumerGroup); } } return(checkpoints); }; async Task <List <EventProcessorCheckpoint> > listLegacyCheckpointsAsync(List <EventProcessorCheckpoint> existingCheckpoints, CancellationToken listCheckpointsToken) { // Legacy checkpoints are not normalized to lowercase var legacyPrefix = string.Format(CultureInfo.InvariantCulture, LegacyCheckpointPrefix, fullyQualifiedNamespace, eventHubName, consumerGroup); var checkpoints = new List <EventProcessorCheckpoint>(); await foreach (BlobItem blob in ContainerClient.GetBlobsAsync(prefix: legacyPrefix, cancellationToken: listCheckpointsToken).ConfigureAwait(false)) { // Skip new checkpoints and empty blobs if (blob.Properties.ContentLength == 0) { continue; } var partitionId = blob.Name.Substring(legacyPrefix.Length); // Check whether there is already a checkpoint for this partition id if (existingCheckpoints.Any(existingCheckpoint => string.Equals(existingCheckpoint.PartitionId, partitionId, StringComparison.Ordinal))) { continue; } var startingPosition = default(EventPosition?); BlobBaseClient blobClient = ContainerClient.GetBlobClient(blob.Name); using var memoryStream = new MemoryStream(); await blobClient.DownloadToAsync(memoryStream, listCheckpointsToken).ConfigureAwait(false); TryReadLegacyCheckpoint( memoryStream.GetBuffer().AsSpan(0, (int)memoryStream.Length), out long?offset, out long?sequenceNumber); if (offset.HasValue) { startingPosition = EventPosition.FromOffset(offset.Value, false); } else if (sequenceNumber.HasValue) { startingPosition = EventPosition.FromSequenceNumber(sequenceNumber.Value, false); } if (startingPosition.HasValue) { checkpoints.Add(new BlobStorageCheckpoint { FullyQualifiedNamespace = fullyQualifiedNamespace, EventHubName = eventHubName, ConsumerGroup = consumerGroup, PartitionId = partitionId, StartingPosition = startingPosition.Value, Offset = offset, SequenceNumber = sequenceNumber }); } else { InvalidCheckpointFound(partitionId, fullyQualifiedNamespace, eventHubName, consumerGroup); } } return(checkpoints); }; List <EventProcessorCheckpoint> checkpoints = null; try { checkpoints = await ApplyRetryPolicy(listCheckpointsAsync, cancellationToken).ConfigureAwait(false); if (InitializeWithLegacyCheckpoints) { checkpoints.AddRange(await ApplyRetryPolicy(ct => listLegacyCheckpointsAsync(checkpoints, ct), cancellationToken).ConfigureAwait(false)); } return(checkpoints); } catch (RequestFailedException ex) when(ex.ErrorCode == BlobErrorCode.ContainerNotFound) { ListCheckpointsError(fullyQualifiedNamespace, eventHubName, consumerGroup, ex); throw new RequestFailedException(BlobsResourceDoesNotExist); } catch (Exception ex) { ListCheckpointsError(fullyQualifiedNamespace, eventHubName, consumerGroup, ex); throw; } finally { ListCheckpointsComplete(fullyQualifiedNamespace, eventHubName, consumerGroup, checkpoints?.Count ?? 0); } }