/// <inheritdoc />
public async Task <bool> UpdatePoisonDataAsync(PoisonData data, CancellationToken cancellationToken)
{
var success = false;
_updateCounter.Increment();
using (Logger.BeginScope("Updating poison message data"))
{
Logger.LogInformation("Updating poison message data for partition {partitionId} with sequence Number {sequenceNumber} and received count of {receivedCount}", data.PartitionId, data.SequenceNumber, data.ReceiveCount);
var blobName = string.Format(_blobNameFormatString, data.PartitionId);
var blobClient = Client.GetBlobClient(blobName);
try
{
success = await SetPoisonMessageContentsAsync(blobClient, data, cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound)
{
Logger.LogError(e, "Error ending with blob not found for partition {partitionId}", data.PartitionId);
throw new RequestFailedException(BlobDoesNotExist);
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.ContainerNotFound)
{
Logger.LogError(e, "Error ending with container not found for partition {partitionId}", data.PartitionId);
await Client.CreateIfNotExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
throw new RequestFailedException(ContainerDoesNotExist);
}
catch (Exception e)
{
Logger.LogError(e, "Unknown error resulted for partition {partitionId}", data.PartitionId);
throw;
}
}
return(success);
}
/// <summary>
/// Updates the poison message information and checks the provided message for poison status, This should only be called on the first message received in a batch, all others should always return false
/// </summary>
/// <param name="data">The event data to check for poisoned message status</param>
/// <param name="partitionContext">The partition context associated with the processor</param>
/// <param name="cancellationToken">A token to monitor for abort requests</param>
/// <returns>True if the message is safe to process, false if poisoned</returns>
protected async Task <bool> UpdatePoisonMonitorStatusAsync(EventData data, ProcessorPartitionContext partitionContext, CancellationToken cancellationToken)
{
var shouldProcess = false;
try
{
Logger.LogInformation("Partition {partitionId} checking if Sequence Number {sequenceNumber} is poisoned", partitionContext.PartitionId, data.SequenceNumber);
if (await _poisonMonitor.IsPoisonedMessageAsync(partitionContext.PartitionId, data.SequenceNumber, cancellationToken).ConfigureAwait(false))
{
var handled = false;
for (var handleIndex = 0; handleIndex < 3 && !handled; handleIndex++)
{
try
{
Logger.LogWarning("Partition {partitionId} found Sequence Number {sequenceNumber} is poisoned", partitionContext.PartitionId, data.SequenceNumber);
handled = await HandlePoisonMessageAsync(data, partitionContext, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
Logger.LogError(e, "Partition {partitionId} error handling poison message {sequenceNumber}", partitionContext.PartitionId, data.SequenceNumber);
}
}
var checkpointSuccess = false;
for (var checkpointAttempt = 0; checkpointAttempt < 3 && !checkpointSuccess; checkpointAttempt++)
{
// Immediately checkpoint to move this forward and do not process further
await _checkpointPolicy.CheckpointAsync(data, true, cancellationToken).ConfigureAwait(false);
checkpointSuccess = true;
}
}
else
{
Logger.LogInformation("Partition {partitionId} Sequence Number {sequenceNumber} is not poisoned", partitionContext.PartitionId, data.SequenceNumber);
shouldProcess = true;
PoisonData poisonData = null;
for (var retrieveAttempt = 0; retrieveAttempt < 3 && poisonData == null; retrieveAttempt++)
{
try
{
poisonData = await _poisonMonitor.GetPoisonDataAsync(partitionContext.PartitionId, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
Logger.LogError(e, "Error retrieving poison message data for partition {partitionId}", partitionContext.PartitionId);
}
}
if (poisonData.SequenceNumber == -1)
{
poisonData.SequenceNumber = data.SequenceNumber;
poisonData.ReceiveCount = 0;
}
else if (poisonData.SequenceNumber == data.SequenceNumber)
{
poisonData.ReceiveCount = poisonData.ReceiveCount + 1;
}
else
{
poisonData.ReceiveCount = 1;
poisonData.SequenceNumber = data.SequenceNumber;
}
var updateSuccess = false;
for (var retrieveAttempt = 0; retrieveAttempt < 3 && !updateSuccess; retrieveAttempt++)
{
try
{
updateSuccess = await _poisonMonitor.UpdatePoisonDataAsync(poisonData, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
Logger.LogError(e, "Error updating poison message data for partition {partitionId}", partitionContext.PartitionId);
}
}
}
}
catch (Exception e)
{
Logger.LogError(e, "Unknown error cannot in poison message handling for partition {partitionId} with sequence Number {sequenceNumber}, ignoring message", partitionContext.PartitionId, data.SequenceNumber);
}
return(shouldProcess);
}
/// <inheritdoc />
protected override bool CheckIfPoisoned(string partitionId, long sequenceNumber, PoisonData data)
{
var poisonDataSequenceNumber = data.SequenceNumber ?? -1;
return(poisonDataSequenceNumber >= 0 && data.ReceiveCount > _allowedReceiveCount && poisonDataSequenceNumber == sequenceNumber);
}
/// <summary>
/// Updates the metadata on the BLOB to reflect the poisoned monitoring data
/// </summary>
/// <param name="blobClient">The BLOB to associate the metadata with</param>
/// <param name="data">The poisoned message tracking contents</param>
/// <param name="cancellationToken">A token to monitor for abort and cancellation requests</param>
/// <returns>True if successful</returns>
private async Task <bool> SetPoisonMessageContentsAsync(BlobClient blobClient, PoisonData data, CancellationToken cancellationToken)
{
var success = false;
var metadata = new Dictionary <string, string>()
{
{ SequenceNumber, (data.SequenceNumber ?? -1L).ToString(CultureInfo.InvariantCulture) },
{ ReceivedCount, data.ReceiveCount.ToString(CultureInfo.InvariantCulture) },
{ PartitionId, data.PartitionId }
};
try
{
using (_updateBlobTiming.Time())
{
await blobClient.SetMetadataAsync(metadata, cancellationToken : cancellationToken).ConfigureAwait(false);
Logger.LogDebug("Metadata set for partition {partitionId}", data.PartitionId);
}
success = true;
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound)
{
Logger.LogInformation("Blob not found partition {partitionId}", data.PartitionId);
using var blobContent = new MemoryStream(Array.Empty <byte>());
using (_updateBlobTiming.Time())
{
await blobClient.UploadAsync(blobContent, metadata : metadata, cancellationToken : cancellationToken).ConfigureAwait(false);
Logger.LogDebug("Blob updated for partition {partitionId}", data.PartitionId);
}
success = true;
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.ContainerNotFound)
{
Logger.LogInformation("Container not found {partitionId}", data.PartitionId);
await Client.CreateIfNotExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
using var blobContent = new MemoryStream(Array.Empty <byte>());
using (_updateBlobTiming.Time())
{
await blobClient.UploadAsync(blobContent, metadata : metadata, cancellationToken : cancellationToken).ConfigureAwait(false);
Logger.LogDebug("Blob uploaded for partition {partitionId}", data.PartitionId);
}
success = true;
}
return(success);
}
/// <summary> /// Used to determine if the current sequence number is considered poison based on the data provided /// </summary> /// <param name="partitionId">The partition id the poison check is associated with</param> /// <param name="sequenceNumber">The sequence number of the message</param> /// <param name="data">The data of the poison monitor</param> /// <returns>True if the message is poisoned</returns> protected abstract bool CheckIfPoisoned(string partitionId, long sequenceNumber, PoisonData data);
/// <inheritdoc />
public async Task <PoisonData> GetPoisonDataAsync(string partitionId, CancellationToken cancellationToken)
{
PoisonData data;
if (!_currentData.TryGetValue(partitionId, out data))
{
data = default;
}
if (data == default)
{
using (Logger.BeginScope("Read Poison Data"))
{
Logger.LogInformation("Reading poison data");
var blobName = string.Format(_blobNameFormatString, partitionId);
var blobClient = Client.GetBlobClient(blobName);
Response <BlobProperties> response = null;
using (_readBlobTiming.Time())
{
try
{
response = await blobClient.GetPropertiesAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound || e.ErrorCode == BlobErrorCode.ContainerNotFound)
{
Logger.LogInformation("Poison monitor data not found for partition {partitionId} attempting to create", partitionId);
await UpdatePoisonDataAsync(new PoisonData
{
PartitionId = partitionId,
ReceiveCount = 0,
SequenceNumber = -1
}, cancellationToken).ConfigureAwait(false);
response = await blobClient.GetPropertiesAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
}
if (response.Value != null)
{
long?sequenceNumber = default;
if (response.Value.Metadata.TryGetValue(SequenceNumber, out var sequence) && long.TryParse(sequence, NumberStyles.Integer, CultureInfo.InvariantCulture, out var sequenceResult))
{
sequenceNumber = sequenceResult;
}
int?receiveCount = default;
if (response.Value.Metadata.TryGetValue(ReceivedCount, out var received) && int.TryParse(received, NumberStyles.Integer, CultureInfo.InvariantCulture, out var receivedResult))
{
receiveCount = receivedResult;
}
data = new PoisonData
{
PartitionId = partitionId,
ReceiveCount = receiveCount ?? 0,
SequenceNumber = sequenceNumber ?? -1L
};
}
}
_currentData.TryAdd(partitionId, data);
}
return(data);
}
/// <inheritdoc />
public Task <bool> UpdatePoisonDataAsync(PoisonData data, CancellationToken cancellationToken)
{
return(Task.FromResult(true));
}
