/// <summary>
/// Updates the metadata on the BLOB to reflect the poisoned monitoring data
/// </summary>
/// <param name="blobClient">The BLOB to associate the metadata with</param>
/// <param name="data">The poisoned message tracking contents</param>
/// <param name="cancellationToken">A token to monitor for abort and cancellation requests</param>
/// <returns>True if successful</returns>
private async Task <bool> SetPoisonMessageContentsAsync(BlobClient blobClient, PoisonData data, CancellationToken cancellationToken)
{
var success = false;
var metadata = new Dictionary <string, string>()
{
{ SequenceNumber, (data.SequenceNumber ?? -1L).ToString(CultureInfo.InvariantCulture) },
{ ReceivedCount, data.ReceiveCount.ToString(CultureInfo.InvariantCulture) },
{ PartitionId, data.PartitionId }
};
try
{
using (_updateBlobTiming.Time())
{
await blobClient.SetMetadataAsync(metadata, cancellationToken : cancellationToken).ConfigureAwait(false);
Logger.LogDebug("Metadata set for partition {partitionId}", data.PartitionId);
}
success = true;
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound)
{
Logger.LogInformation("Blob not found partition {partitionId}", data.PartitionId);
using var blobContent = new MemoryStream(Array.Empty <byte>());
using (_updateBlobTiming.Time())
{
await blobClient.UploadAsync(blobContent, metadata : metadata, cancellationToken : cancellationToken).ConfigureAwait(false);
Logger.LogDebug("Blob updated for partition {partitionId}", data.PartitionId);
}
success = true;
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.ContainerNotFound)
{
Logger.LogInformation("Container not found {partitionId}", data.PartitionId);
await Client.CreateIfNotExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
using var blobContent = new MemoryStream(Array.Empty <byte>());
using (_updateBlobTiming.Time())
{
await blobClient.UploadAsync(blobContent, metadata : metadata, cancellationToken : cancellationToken).ConfigureAwait(false);
Logger.LogDebug("Blob uploaded for partition {partitionId}", data.PartitionId);
}
success = true;
}
return(success);
}
/// <summary>
/// Updates the checkpoint for the partition
/// </summary>
/// <param name="eventData">The event data to checkpoint to</param>
/// <param name="cancellationToken">A token to monitor for abort requests.</param>
/// <returns>True if the checkpointing was successful.</returns>
public async Task <bool> CheckpointAsync(EventData eventData, CancellationToken cancellationToken)
{
var success = false;
try
{
_logger.LogInformation("Checkpointing for partition {partitionId}", PartitionId);
_checkpointCounter.Increment();
using (_checkpointTiming.Time())
{
await _checkpointManager.UpdateCheckpointAsync(eventData, this, cancellationToken).ConfigureAwait(false);
}
success = true;
_logger.LogDebug("Finished checkpointing for partition {partitionId}", PartitionId);
}
catch (Exception e)
{
_logger.LogError(e, "Error checkpointing for partition {partitionId}", PartitionId);
}
return(success);
}
/// <summary>
/// Authenticates the client credentials against the hub
/// </summary>
/// <param name="request">The request</param>
/// <param name="cancellationToken">A token to monitor for abort requests</param>
/// <returns>The authentication response</returns>
protected virtual async Task <AuthenticationResponsePayload> AuthenticateClientAsync(HttpRequest request, CancellationToken cancellationToken)
{
AuthenticationResponsePayload response = null;
using (var reader = new StreamReader(request.Body))
{
var requestPayload = JsonConvert.DeserializeObject <AuthenticationRequestPayload>(reader.ReadToEnd());
using (_authenticationTiming.Time())
{
if (await AuthenticateAsync(request, requestPayload, cancellationToken).ConfigureAwait(false))
{
var accessToken = SharedAccessTokens.GenerateSasToken($"{_sasTokenScheme}://{requestPayload.UserName}", _sasSigningKey, _sasTokenPolicyName, (int)_sasTokenTimeout.TotalSeconds);
response = new AuthenticationResponsePayload
{
TokenType = "bearer",
AccessToken = accessToken
};
}
}
}
return(response);
}
/// <inheritdoc />
public async Task <EpochOperationResult> AddOrUpdateEpochAsync(EpochValue value, bool force, CancellationToken cancellationToken)
{
EpochOperationResult results;
using (_logger.BeginScope("Update epoch"))
{
_logger.LogInformation("Adding epoch: {partitionId}, forced: {force}", value.PartitionId, force);
CloudBlockBlob epochBlob = _consumerGroupDirectory.GetBlockBlobReference(value.PartitionId);
try
{
_epochUpdateCounter.Increment();
AccessCondition accessCondition = force ? AccessCondition.GenerateLeaseCondition("*") : AccessCondition.GenerateLeaseCondition(value.ConcurrencyValue);
var content = JsonConvert.SerializeObject(value);
_logger.LogDebug("Writing content to partition {partitionId}, raw: {json}", value.PartitionId, content);
using (_storagePerformanceSummary.Time())
{
await epochBlob.UploadTextAsync(content, _leaseEncoding, accessCondition, _defaultRequestOptions, null).ConfigureAwait(false);
}
value.ConcurrencyValue = epochBlob.Properties.ETag;
_logger.LogDebug("Completed write of epoch to partition {partitionId}", value.PartitionId);
results = EpochOperationResult.Success;
}
catch (StorageException e) when(e.RequestInformation.HttpStatusCode == (int)HttpStatusCode.Conflict)
{
_logger.LogInformation("Epoch content conflict for partition {partitionId}", value.PartitionId);
results = EpochOperationResult.Conflict;
_epochErrorCounter.Increment();
}
catch (Exception e)
{
_epochErrorCounter.Increment();
results = EpochOperationResult.Failure;
_logger.LogError(e, "Error adding or updating epoch for partition {partitionId}", value.PartitionId);
}
return(results);
}
}
/// <inheritdoc />
public virtual async Task <IEnumerable <EventProcessorCheckpoint> > ListCheckpointsAsync(CancellationToken cancellationToken)
{
var checkpoints = new List <EventProcessorCheckpoint>();
_listCheckpointsCounter.Increment();
using (Logger.BeginScope("Listing checkpoints"))
{
// Timing added around loop although not ideal. This is tied the paging async
using (_listBlobTiming.Time())
{
Logger.LogInformation("Retrieving checkpoints");
await foreach (BlobItem blob in Client.GetBlobsAsync(traits: BlobTraits.Metadata, prefix: _checkpointPrefix, cancellationToken: cancellationToken).ConfigureAwait(false))
{
var partitionId = blob.Name.Substring(_checkpointPrefix.Length + 1);
var startingPosition = default(EventPosition?);
Logger.LogDebug("Retrieved blob for partition {partitionId}", partitionId);
var sequenceNumber = await GetCheckpointSequenceNumberAsync(partitionId, blob, cancellationToken).ConfigureAwait(false);
if (sequenceNumber.HasValue)
{
Logger.LogDebug("Checkpoint was found with a sequence number of {sequenceNumber}", sequenceNumber);
startingPosition = EventPosition.FromSequenceNumber(sequenceNumber.Value, false);
checkpoints.Add(new Checkpoint
{
FullyQualifiedNamespace = _processorClient.FullyQualifiedNamespace,
EventHubName = _processorClient.EventHubName,
ConsumerGroup = _processorClient.ConsumerGroup,
PartitionId = partitionId,
StartingPosition = startingPosition.Value,
SequenceNumber = sequenceNumber
});
}
else
{
Logger.LogError("An invalid checkpoint was found, no starting position");
}
}
}
}
return(checkpoints);
}
/// <summary>
/// Processes the web socket session
/// </summary>
/// <param name="socket">The web socket that is in use</param>
/// <param name="cancellationToken">A token to monitor for abort requests</param>
public async Task ProcessAsync(WebSocket socket, CancellationToken cancellationToken)
{
using (Logger.BeginScope("Hander Process"))
{
Logger.LogInformation("Web socket handlers invoked");
using (_serviceSessionGauge.TrackExecution())
{
Task receiverTask = null;
try
{
_socket = socket;
LastActivityTime = DateTime.UtcNow;
Logger.LogDebug("Web socket receiver starting");
receiverTask = ReceiveMessagesAsync(socket, cancellationToken);
using (_sessionTimes.Time())
{
while (SocketState == WebSocketState.Open && LastActivityTime.Add(WebSocketConfiguration.SessionTimeout ?? TimeSpan.FromSeconds(30)) > DateTime.UtcNow)
{
await IterationAsync(ContainerLifecycle.CancellationToken).ConfigureAwait(false);
}
}
Logger.LogInformation("Web socket session completed");
}
catch (Exception e)
{
Logger.LogError(e, "Error processing session, aborting");
_errorCounter.Increment();
}
Logger.LogDebug("Web socket iteration complete, closing");
await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Session closed", CancellationToken.None).ConfigureAwait(false);
if (receiverTask != null)
{
await receiverTask.ConfigureAwait(false);
}
}
Logger.LogInformation("Web socket handlers ending");
}
}
/// <summary>
/// Receive loop for getting messages
/// </summary>
/// <param name="socket">The web socket that is associated with the session</param>
/// <param name="cancellationToken">A token to monitor for abort requests</param>
private async Task ReceiveMessagesAsync(WebSocket socket, CancellationToken cancellationToken)
{
using (Logger.BeginScope("Receive Message Loop"))
{
var buffer = BufferPool.Take();
var bufferList = new List <(byte[], int)>();
Task <WebSocketReceiveResult> receiveTask = null;
while (SocketState == WebSocketState.Open)
{
try
{
if (receiveTask == null)
{
receiveTask = socket.ReceiveAsync(buffer, CancellationToken.None);
}
var completedTask = await Task.WhenAny(receiveTask, Task.Delay(250)).ConfigureAwait(false);
if (receiveTask.IsCompleted)
{
var result = receiveTask.Result;
LastActivityTime = DateTime.UtcNow;
if (result.Count > 0)
{
bufferList.Add((buffer, result.Count));
buffer = BufferPool.Take();
}
if (result.EndOfMessage)
{
var byteCount = 0;
foreach (var currentBuffer in bufferList)
{
byteCount += currentBuffer.Item2;
}
var finalBuffer = new byte[byteCount];
var position = 0;
foreach (var currentBuffer in bufferList)
{
Buffer.BlockCopy(currentBuffer.Item1, 0, finalBuffer, position, currentBuffer.Item2);
position += currentBuffer.Item2;
}
foreach (var currentBuffer in bufferList)
{
BufferPool.Return(currentBuffer.Item1);
}
bufferList.Clear();
try
{
using (_receiveTimes.Time())
{
await MessageReceivedAsync(finalBuffer, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception e)
{
_errorCounter.Increment();
Logger.LogError(e, "Error processing received message");
}
}
receiveTask = null;
}
}
catch (Exception e)
{
Logger.LogError(e, "Error in receive loop");
_errorCounter.Increment();
if (bufferList.Count > 0)
{
foreach (var currentBuffer in bufferList)
{
BufferPool.Return(currentBuffer.Item1);
}
}
}
}
if (bufferList.Count > 0)
{
foreach (var currentBuffer in bufferList)
{
BufferPool.Return(currentBuffer.Item1);
}
}
BufferPool.Return(buffer);
}
}
/// <inheritdoc />
public async Task <bool> CreateLeaseStoreIfNotExistsAsync()
{
bool success;
using (_logger.BeginScope("Create lease store"))
{
_logger.LogInformation("Creating Azure Storage lease store");
using (_storagePerformanceSummary.Time())
{
success = await _leaseStoreContainer.CreateIfNotExistsAsync(_defaultRequestOptions, _operationContext).ConfigureAwait(false);
_logger.LogDebug("Lease store creation results {result}", success);
}
}
return(success);
}
/// <inheritdoc />
public async Task <PoisonData> GetPoisonDataAsync(string partitionId, CancellationToken cancellationToken)
{
PoisonData data;
if (!_currentData.TryGetValue(partitionId, out data))
{
data = default;
}
if (data == default)
{
using (Logger.BeginScope("Read Poison Data"))
{
Logger.LogInformation("Reading poison data");
var blobName = string.Format(_blobNameFormatString, partitionId);
var blobClient = Client.GetBlobClient(blobName);
Response <BlobProperties> response = null;
using (_readBlobTiming.Time())
{
try
{
response = await blobClient.GetPropertiesAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException e) when(e.ErrorCode == BlobErrorCode.BlobNotFound || e.ErrorCode == BlobErrorCode.ContainerNotFound)
{
Logger.LogInformation("Poison monitor data not found for partition {partitionId} attempting to create", partitionId);
await UpdatePoisonDataAsync(new PoisonData
{
PartitionId = partitionId,
ReceiveCount = 0,
SequenceNumber = -1
}, cancellationToken).ConfigureAwait(false);
response = await blobClient.GetPropertiesAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
}
if (response.Value != null)
{
long?sequenceNumber = default;
if (response.Value.Metadata.TryGetValue(SequenceNumber, out var sequence) && long.TryParse(sequence, NumberStyles.Integer, CultureInfo.InvariantCulture, out var sequenceResult))
{
sequenceNumber = sequenceResult;
}
int?receiveCount = default;
if (response.Value.Metadata.TryGetValue(ReceivedCount, out var received) && int.TryParse(received, NumberStyles.Integer, CultureInfo.InvariantCulture, out var receivedResult))
{
receiveCount = receivedResult;
}
data = new PoisonData
{
PartitionId = partitionId,
ReceiveCount = receiveCount ?? 0,
SequenceNumber = sequenceNumber ?? -1L
};
}
}
_currentData.TryAdd(partitionId, data);
}
return(data);
}
/// <inheritdoc />
public async Task <bool> CheckpointStoreExistsAsync()
{
var success = false;
using (_logger.BeginScope("Checkpoint Store Exists"))
{
_logger.LogDebug("Check if store exists");
using (_storagePerformanceSummary.Time())
{
success = await _checkpointStoreContainer.ExistsAsync(_defaultRequestOptions, AzureBlobCommon.DefaultOperationContext).ConfigureAwait(false);
}
_logger.LogDebug("Check if store exists result {result}", success);
}
return(success);
}