/// <inheritdoc /> public async Task <bool> UpdatePoisonDataAsync(PoisonData data, CancellationToken cancellationToken) { var success = false; _updateCounter.Increment(); using (Logger.BeginScope("Updating poison message data")) { Logger.LogInformation("Updating poison message data for partition {partitionId} with sequence Number {sequenceNumber} and received count of {receivedCount}", data.PartitionId, data.SequenceNumber, data.ReceiveCount); var blobName = string.Format(_blobNameFormatString, data.PartitionId); var blobClient = Client.GetBlobClient(blobName); try { success = await SetPoisonMessageContentsAsync(blobClient, data, cancellationToken).ConfigureAwait(false); } catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound) { Logger.LogError(e, "Error ending with blob not found for partition {partitionId}", data.PartitionId); throw new RequestFailedException(BlobDoesNotExist); } catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.ContainerNotFound) { Logger.LogError(e, "Error ending with container not found for partition {partitionId}", data.PartitionId); await Client.CreateIfNotExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false); throw new RequestFailedException(ContainerDoesNotExist); } catch (Exception e) { Logger.LogError(e, "Unknown error resulted for partition {partitionId}", data.PartitionId); throw; } } return(success); }
/// <summary> /// Updates the poison message information and checks the provided message for poison status, This should only be called on the first message received in a batch, all others should always return false /// </summary> /// <param name="data">The event data to check for poisoned message status</param> /// <param name="partitionContext">The partition context associated with the processor</param> /// <param name="cancellationToken">A token to monitor for abort requests</param> /// <returns>True if the message is safe to process, false if poisoned</returns> protected async Task <bool> UpdatePoisonMonitorStatusAsync(EventData data, ProcessorPartitionContext partitionContext, CancellationToken cancellationToken) { var shouldProcess = false; try { Logger.LogInformation("Partition {partitionId} checking if Sequence Number {sequenceNumber} is poisoned", partitionContext.PartitionId, data.SequenceNumber); if (await _poisonMonitor.IsPoisonedMessageAsync(partitionContext.PartitionId, data.SequenceNumber, cancellationToken).ConfigureAwait(false)) { var handled = false; for (var handleIndex = 0; handleIndex < 3 && !handled; handleIndex++) { try { Logger.LogWarning("Partition {partitionId} found Sequence Number {sequenceNumber} is poisoned", partitionContext.PartitionId, data.SequenceNumber); handled = await HandlePoisonMessageAsync(data, partitionContext, cancellationToken).ConfigureAwait(false); } catch (Exception e) { Logger.LogError(e, "Partition {partitionId} error handling poison message {sequenceNumber}", partitionContext.PartitionId, data.SequenceNumber); } } var checkpointSuccess = false; for (var checkpointAttempt = 0; checkpointAttempt < 3 && !checkpointSuccess; checkpointAttempt++) { // Immediately checkpoint to move this forward and do not process further await _checkpointPolicy.CheckpointAsync(data, true, cancellationToken).ConfigureAwait(false); checkpointSuccess = true; } } else { Logger.LogInformation("Partition {partitionId} Sequence Number {sequenceNumber} is not poisoned", partitionContext.PartitionId, data.SequenceNumber); shouldProcess = true; PoisonData poisonData = null; for (var retrieveAttempt = 0; retrieveAttempt < 3 && poisonData == null; retrieveAttempt++) { try { poisonData = await _poisonMonitor.GetPoisonDataAsync(partitionContext.PartitionId, cancellationToken).ConfigureAwait(false); } catch (Exception e) { Logger.LogError(e, "Error retrieving poison message data for partition {partitionId}", partitionContext.PartitionId); } } if (poisonData.SequenceNumber == -1) { poisonData.SequenceNumber = data.SequenceNumber; poisonData.ReceiveCount = 0; } else if (poisonData.SequenceNumber == data.SequenceNumber) { poisonData.ReceiveCount = poisonData.ReceiveCount + 1; } else { poisonData.ReceiveCount = 1; poisonData.SequenceNumber = data.SequenceNumber; } var updateSuccess = false; for (var retrieveAttempt = 0; retrieveAttempt < 3 && !updateSuccess; retrieveAttempt++) { try { updateSuccess = await _poisonMonitor.UpdatePoisonDataAsync(poisonData, cancellationToken).ConfigureAwait(false); } catch (Exception e) { Logger.LogError(e, "Error updating poison message data for partition {partitionId}", partitionContext.PartitionId); } } } } catch (Exception e) { Logger.LogError(e, "Unknown error cannot in poison message handling for partition {partitionId} with sequence Number {sequenceNumber}, ignoring message", partitionContext.PartitionId, data.SequenceNumber); } return(shouldProcess); }
/// <inheritdoc /> protected override bool CheckIfPoisoned(string partitionId, long sequenceNumber, PoisonData data) { var poisonDataSequenceNumber = data.SequenceNumber ?? -1; return(poisonDataSequenceNumber >= 0 && data.ReceiveCount > _allowedReceiveCount && poisonDataSequenceNumber == sequenceNumber); }
/// <summary> /// Updates the metadata on the BLOB to reflect the poisoned monitoring data /// </summary> /// <param name="blobClient">The BLOB to associate the metadata with</param> /// <param name="data">The poisoned message tracking contents</param> /// <param name="cancellationToken">A token to monitor for abort and cancellation requests</param> /// <returns>True if successful</returns> private async Task <bool> SetPoisonMessageContentsAsync(BlobClient blobClient, PoisonData data, CancellationToken cancellationToken) { var success = false; var metadata = new Dictionary <string, string>() { { SequenceNumber, (data.SequenceNumber ?? -1L).ToString(CultureInfo.InvariantCulture) }, { ReceivedCount, data.ReceiveCount.ToString(CultureInfo.InvariantCulture) }, { PartitionId, data.PartitionId } }; try { using (_updateBlobTiming.Time()) { await blobClient.SetMetadataAsync(metadata, cancellationToken : cancellationToken).ConfigureAwait(false); Logger.LogDebug("Metadata set for partition {partitionId}", data.PartitionId); } success = true; } catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound) { Logger.LogInformation("Blob not found partition {partitionId}", data.PartitionId); using var blobContent = new MemoryStream(Array.Empty <byte>()); using (_updateBlobTiming.Time()) { await blobClient.UploadAsync(blobContent, metadata : metadata, cancellationToken : cancellationToken).ConfigureAwait(false); Logger.LogDebug("Blob updated for partition {partitionId}", data.PartitionId); } success = true; } catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.ContainerNotFound) { Logger.LogInformation("Container not found {partitionId}", data.PartitionId); await Client.CreateIfNotExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false); using var blobContent = new MemoryStream(Array.Empty <byte>()); using (_updateBlobTiming.Time()) { await blobClient.UploadAsync(blobContent, metadata : metadata, cancellationToken : cancellationToken).ConfigureAwait(false); Logger.LogDebug("Blob uploaded for partition {partitionId}", data.PartitionId); } success = true; } return(success); }
/// <summary> /// Used to determine if the current sequence number is considered poison based on the data provided /// </summary> /// <param name="partitionId">The partition id the poison check is associated with</param> /// <param name="sequenceNumber">The sequence number of the message</param> /// <param name="data">The data of the poison monitor</param> /// <returns>True if the message is poisoned</returns> protected abstract bool CheckIfPoisoned(string partitionId, long sequenceNumber, PoisonData data);
/// <inheritdoc /> public async Task <PoisonData> GetPoisonDataAsync(string partitionId, CancellationToken cancellationToken) { PoisonData data; if (!_currentData.TryGetValue(partitionId, out data)) { data = default; } if (data == default) { using (Logger.BeginScope("Read Poison Data")) { Logger.LogInformation("Reading poison data"); var blobName = string.Format(_blobNameFormatString, partitionId); var blobClient = Client.GetBlobClient(blobName); Response <BlobProperties> response = null; using (_readBlobTiming.Time()) { try { response = await blobClient.GetPropertiesAsync(cancellationToken : cancellationToken).ConfigureAwait(false); } catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound || e.ErrorCode == BlobErrorCode.ContainerNotFound) { Logger.LogInformation("Poison monitor data not found for partition {partitionId} attempting to create", partitionId); await UpdatePoisonDataAsync(new PoisonData { PartitionId = partitionId, ReceiveCount = 0, SequenceNumber = -1 }, cancellationToken).ConfigureAwait(false); response = await blobClient.GetPropertiesAsync(cancellationToken : cancellationToken).ConfigureAwait(false); } } if (response.Value != null) { long?sequenceNumber = default; if (response.Value.Metadata.TryGetValue(SequenceNumber, out var sequence) && long.TryParse(sequence, NumberStyles.Integer, CultureInfo.InvariantCulture, out var sequenceResult)) { sequenceNumber = sequenceResult; } int?receiveCount = default; if (response.Value.Metadata.TryGetValue(ReceivedCount, out var received) && int.TryParse(received, NumberStyles.Integer, CultureInfo.InvariantCulture, out var receivedResult)) { receiveCount = receivedResult; } data = new PoisonData { PartitionId = partitionId, ReceiveCount = receiveCount ?? 0, SequenceNumber = sequenceNumber ?? -1L }; } } _currentData.TryAdd(partitionId, data); } return(data); }
/// <inheritdoc /> public Task <bool> UpdatePoisonDataAsync(PoisonData data, CancellationToken cancellationToken) { return(Task.FromResult(true)); }