/// <inheritdoc /> public async Task UpdateCheckpointAsync(Lease lease, Checkpoint checkpoint) { using (_logger.BeginScope("Updating checkpoint")) { _checkpointUpdateCounter.Increment(); CheckpointLease blobLease = new CheckpointLease(lease) { Offset = checkpoint.Offset, SequenceNumber = checkpoint.SequenceNumber, PartitionId = checkpoint.PartitionId }; _logger.LogInformation("Updating checkpoint: Partition Id: {partitionId}", blobLease.PartitionId); CloudBlockBlob leaseBlob = _consumerGroupDirectory.GetBlockBlobReference(blobLease.PartitionId); try { string jsonToUpload = JsonConvert.SerializeObject(lease); _logger.LogTrace("Updating checkpoint: Partition Id: {partitionId}, RawJson: {json}", blobLease.PartitionId, jsonToUpload); AccessCondition accessCondition; if (_etags.ContainsKey(lease.PartitionId)) { accessCondition = AccessCondition.GenerateIfMatchCondition(_etags[lease.PartitionId]); } else { accessCondition = AzureBlobCommon.DefaultAccessCondition; } using (_storagePerformanceSummary.Time()) { await leaseBlob.UploadTextAsync(jsonToUpload, _checkpointEncoding, accessCondition, _defaultRequestOptions, AzureBlobCommon.DefaultOperationContext).ConfigureAwait(false); } _etags.AddOrUpdate(lease.PartitionId, leaseBlob.Properties.ETag, (pid, petag) => leaseBlob.Properties.ETag); _logger.LogInformation("Updated checkpoint for partition {partitionId}", blobLease.PartitionId); } catch (StorageException e) { _checkpointErrorCounter.Increment(); _logger.LogError(e, "Error updating partition {partitionId}", blobLease.PartitionId); throw; } } }
/// <summary> /// An common method to retrieve the Azure BLOB Lease from BLOB store /// </summary> /// <param name="partitionId">The partition id to retrieve the checkpoint for</param> /// <returns>The checkpoint with contents of the BLOB and the etag</returns> private async Task <(CheckpointLease lease, string etag)> GetCheckpointInternalAsync(string partitionId) { using (_logger.BeginScope("Get checkpoint internal")) { CloudBlockBlob leaseBlob = _consumerGroupDirectory.GetBlockBlobReference(partitionId); string jsonCheckpoint; using (_storagePerformanceSummary.Time()) { jsonCheckpoint = await leaseBlob.DownloadTextAsync(_checkpointEncoding, AzureBlobCommon.DefaultAccessCondition, _defaultRequestOptions, AzureBlobCommon.DefaultOperationContext).ConfigureAwait(false); } _logger.LogDebug("Retrieved checkpoint for partition {id} :: {json}", partitionId, jsonCheckpoint); CheckpointLease lease = JsonConvert.DeserializeObject <CheckpointLease>(jsonCheckpoint); _checkpointReadCounter.Increment(); return(new CheckpointLease(lease), leaseBlob.Properties.ETag); } }
/// <inheritdoc /> public async Task <Checkpoint> CreateCheckpointIfNotExistsAsync(string partitionId) { using (_logger.BeginScope("Create checkpoint")) { _checkpointUpdateCounter.Increment(); CheckpointLease checkpoint; _logger.LogInformation("Creating checkpoint for partition {partitionId}", partitionId); try { checkpoint = new CheckpointLease(partitionId); string json = JsonConvert.SerializeObject(checkpoint); CloudBlockBlob checkpointBlob = _consumerGroupDirectory.GetBlockBlobReference(partitionId); bool checkpointExists; using (_storagePerformanceSummary.Time()) { checkpointExists = await checkpointBlob.ExistsAsync().ConfigureAwait(false); } if (!checkpointExists) { _logger.LogDebug("Checkpoint does not exist for partition {partitionId}", partitionId); using (_storagePerformanceSummary.Time()) { await checkpointBlob.UploadTextAsync(json, _checkpointEncoding, AzureBlobCommon.OverwriteAccessCondition, _defaultRequestOptions, AzureBlobCommon.DefaultOperationContext).ConfigureAwait(false); } _logger.LogDebug("Uploaded checkpoint information for partition {partitionId}", partitionId); } else { _logger.LogDebug("Checkpoint exists, retrieving checkpoint for partition {partitionId}", partitionId); var(lease, etag) = await GetCheckpointInternalAsync(partitionId).ConfigureAwait(false); _logger.LogDebug("Checkpoint retrieved for partition {partitionId}", partitionId); checkpoint = lease; _etags.AddOrUpdate(partitionId, etag, (pid, petag) => etag); } } catch (StorageException e) { if (e.RequestInformation.ErrorCode == BlobErrorCodeStrings.BlobAlreadyExists || e.RequestInformation.ErrorCode == BlobErrorCodeStrings.LeaseIdMissing) { // The blob already exists. _logger.LogInformation("Checkpoint already exists, Partition {patitionId}", partitionId); var(lease, etag) = await GetCheckpointInternalAsync(partitionId).ConfigureAwait(false); checkpoint = lease; _etags.AddOrUpdate(partitionId, etag, (pid, petag) => etag); } else { _checkpointErrorCounter.Increment(); _logger.LogError(e, "CreateCheckpointIfNotExist StorageException, Partition {patitionId}", partitionId); throw; } } return(checkpoint); } }