/// <inheritdoc />
public async Task UpdateCheckpointAsync(Lease lease, Checkpoint checkpoint)
{
using (_logger.BeginScope("Updating checkpoint"))
{
_checkpointUpdateCounter.Increment();
CheckpointLease blobLease = new CheckpointLease(lease)
{
Offset = checkpoint.Offset,
SequenceNumber = checkpoint.SequenceNumber,
PartitionId = checkpoint.PartitionId
};
_logger.LogInformation("Updating checkpoint: Partition Id: {partitionId}", blobLease.PartitionId);
CloudBlockBlob leaseBlob = _consumerGroupDirectory.GetBlockBlobReference(blobLease.PartitionId);
try
{
string jsonToUpload = JsonConvert.SerializeObject(lease);
_logger.LogTrace("Updating checkpoint: Partition Id: {partitionId}, RawJson: {json}", blobLease.PartitionId, jsonToUpload);
AccessCondition accessCondition;
if (_etags.ContainsKey(lease.PartitionId))
{
accessCondition = AccessCondition.GenerateIfMatchCondition(_etags[lease.PartitionId]);
}
else
{
accessCondition = AzureBlobCommon.DefaultAccessCondition;
}
using (_storagePerformanceSummary.Time())
{
await leaseBlob.UploadTextAsync(jsonToUpload, _checkpointEncoding, accessCondition, _defaultRequestOptions, AzureBlobCommon.DefaultOperationContext).ConfigureAwait(false);
}
_etags.AddOrUpdate(lease.PartitionId, leaseBlob.Properties.ETag, (pid, petag) => leaseBlob.Properties.ETag);
_logger.LogInformation("Updated checkpoint for partition {partitionId}", blobLease.PartitionId);
}
catch (StorageException e)
{
_checkpointErrorCounter.Increment();
_logger.LogError(e, "Error updating partition {partitionId}", blobLease.PartitionId);
throw;
}
}
}
/// <summary>
/// An common method to retrieve the Azure BLOB Lease from BLOB store
/// </summary>
/// <param name="partitionId">The partition id to retrieve the checkpoint for</param>
/// <returns>The checkpoint with contents of the BLOB and the etag</returns>
private async Task <(CheckpointLease lease, string etag)> GetCheckpointInternalAsync(string partitionId)
{
using (_logger.BeginScope("Get checkpoint internal"))
{
CloudBlockBlob leaseBlob = _consumerGroupDirectory.GetBlockBlobReference(partitionId);
string jsonCheckpoint;
using (_storagePerformanceSummary.Time())
{
jsonCheckpoint = await leaseBlob.DownloadTextAsync(_checkpointEncoding, AzureBlobCommon.DefaultAccessCondition, _defaultRequestOptions, AzureBlobCommon.DefaultOperationContext).ConfigureAwait(false);
}
_logger.LogDebug("Retrieved checkpoint for partition {id} :: {json}", partitionId, jsonCheckpoint);
CheckpointLease lease = JsonConvert.DeserializeObject <CheckpointLease>(jsonCheckpoint);
_checkpointReadCounter.Increment();
return(new CheckpointLease(lease), leaseBlob.Properties.ETag);
}
}
/// <inheritdoc />
public async Task <Checkpoint> CreateCheckpointIfNotExistsAsync(string partitionId)
{
using (_logger.BeginScope("Create checkpoint"))
{
_checkpointUpdateCounter.Increment();
CheckpointLease checkpoint;
_logger.LogInformation("Creating checkpoint for partition {partitionId}", partitionId);
try
{
checkpoint = new CheckpointLease(partitionId);
string json = JsonConvert.SerializeObject(checkpoint);
CloudBlockBlob checkpointBlob = _consumerGroupDirectory.GetBlockBlobReference(partitionId);
bool checkpointExists;
using (_storagePerformanceSummary.Time())
{
checkpointExists = await checkpointBlob.ExistsAsync().ConfigureAwait(false);
}
if (!checkpointExists)
{
_logger.LogDebug("Checkpoint does not exist for partition {partitionId}", partitionId);
using (_storagePerformanceSummary.Time())
{
await checkpointBlob.UploadTextAsync(json, _checkpointEncoding, AzureBlobCommon.OverwriteAccessCondition, _defaultRequestOptions, AzureBlobCommon.DefaultOperationContext).ConfigureAwait(false);
}
_logger.LogDebug("Uploaded checkpoint information for partition {partitionId}", partitionId);
}
else
{
_logger.LogDebug("Checkpoint exists, retrieving checkpoint for partition {partitionId}", partitionId);
var(lease, etag) = await GetCheckpointInternalAsync(partitionId).ConfigureAwait(false);
_logger.LogDebug("Checkpoint retrieved for partition {partitionId}", partitionId);
checkpoint = lease;
_etags.AddOrUpdate(partitionId, etag, (pid, petag) => etag);
}
}
catch (StorageException e)
{
if (e.RequestInformation.ErrorCode == BlobErrorCodeStrings.BlobAlreadyExists || e.RequestInformation.ErrorCode == BlobErrorCodeStrings.LeaseIdMissing)
{
// The blob already exists.
_logger.LogInformation("Checkpoint already exists, Partition {patitionId}", partitionId);
var(lease, etag) = await GetCheckpointInternalAsync(partitionId).ConfigureAwait(false);
checkpoint = lease;
_etags.AddOrUpdate(partitionId, etag, (pid, petag) => etag);
}
else
{
_checkpointErrorCounter.Increment();
_logger.LogError(e, "CreateCheckpointIfNotExist StorageException, Partition {patitionId}", partitionId);
throw;
}
}
return(checkpoint);
}
}
