private async Task <bool> TryHandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken)
{
bool handled = true;
try
{
var resource = (T)resourceEvent.Resource;
if (IsDeletePending(resource))
{
await HandleDeletedEventAsync(resource, cancellationToken);
}
else
{
await HandleAddedOrModifiedEventAsync(resource, cancellationToken);
}
}
catch (OperationCanceledException)
{
_logger.LogDebug($"Canceled HandleEvent, {resourceEvent}");
}
catch (Exception exception)
{
if (exception is HttpOperationException httpException && httpException.Response?.StatusCode == System.Net.HttpStatusCode.Conflict)
{
// Conflicts happen. The next event will make the resource consistent again
_logger.LogDebug(exception, $"Conflict handling {resourceEvent}");
}
/// <summary>
/// Track the end of an event handling
/// </summary>
public void EndHandleEvent(CustomResourceEvent resourceEvent)
{
lock (this)
{
_logger.LogTrace($"EndHandleEvent {resourceEvent}");
_handling.Remove(resourceEvent.ResourceUid);
}
}
/// <summary>
/// Track the begin of an event handling
/// </summary>
public void BeginHandleEvent(CustomResourceEvent resourceEvent)
{
lock (this)
{
_logger.LogTrace($"BeginHandleEvent {resourceEvent}");
_handling[resourceEvent.ResourceUid] = resourceEvent;
}
}
/// <summary>
/// Enqueue the event
/// </summary>
public void Enqueue(CustomResourceEvent resourceEvent)
{
lock (this)
{
_logger.LogTrace($"Enqueue {resourceEvent}");
// Insert or update the next event for the resource
_queuesByResource[resourceEvent.ResourceUid] = resourceEvent;
}
}
/// <summary>
/// Dispatches an incoming event to the controller
/// </summary>
public void OnIncomingEvent(WatchEventType eventType, CustomResource resource)
{
var resourceEvent = new CustomResourceEvent(eventType, resource);
_logger.LogDebug($"Received event {resourceEvent}");
Controller.ProcessEventAsync(resourceEvent, _cancellationToken)
.ContinueWith(t =>
{
if (t.IsFaulted)
{
var exception = t.Exception.Flatten().InnerException;
_logger.LogError(exception, $"Error processing {resourceEvent}");
}
});
}
private async Task HandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken)
{
if (resourceEvent == null)
{
_logger.LogWarning($"Skip HandleEvent, {nameof(resourceEvent)} is null");
return;
}
_logger.LogDebug($"Begin HandleEvent, {resourceEvent}");
_eventManager.BeginHandleEvent(resourceEvent);
var attempt = 1;
var delay = RetryPolicy.InitialDelay;
while (true)
{
// Try to handle the event
var handled = await TryHandleEventAsync(resourceEvent, cancellationToken);
if (handled)
{
break;
}
// Something went wrong
if (!CanTryAgain(resourceEvent, attempt, cancellationToken))
{
break;
}
_logger.LogDebug($"Retrying to handle {resourceEvent} in {delay}ms (attempt #{attempt})");
// Wait
await Task.Delay(delay);
// Increase the delay for the next attempt
attempt++;
delay = (int)(delay * RetryPolicy.DelayMultiplier);
}
_logger.LogDebug($"End HandleEvent, {resourceEvent}");
_eventManager.EndHandleEvent(resourceEvent);
}
/// <summary>
/// Processes a custom resource event
/// </summary>
/// <param name="resourceEvent">The event to handle</param>
/// <param name="cancellationToken">Signals if the current execution has been canceled</param>
public async Task ProcessEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken)
{
_logger.LogDebug($"Begin ProcessEvent, {resourceEvent}");
if (resourceEvent.Type == WatchEventType.Error)
{
_logger.LogError($"Received Error event, {resourceEvent.Resource}");
return;
}
if (resourceEvent.Type == WatchEventType.Deleted)
{
// Skip Deleted events since there is nothing else to do
_logger.LogDebug($"Skip ProcessEvent, received Deleted event, {resourceEvent.Resource}");
return;
}
if (resourceEvent.Type == WatchEventType.Bookmark)
{
// Skip Bookmark events since there is nothing else to do
_logger.LogDebug($"Skip ProcessEvent, received Bookmark event, {resourceEvent.Resource}");
return;
}
// Enqueue the event
_eventManager.Enqueue(resourceEvent);
while (!cancellationToken.IsCancellationRequested)
{
// Dequeue the next event to process for this resource, if any
var nextEvent = _eventManager.Dequeue(resourceEvent.ResourceUid);
if (nextEvent == null)
{
break;
}
await HandleEventAsync(nextEvent, cancellationToken);
}
_logger.LogDebug($"End ProcessEvent, {resourceEvent}");
}
private bool CanTryAgain(CustomResourceEvent resourceEvent, int attemptNumber, CancellationToken cancellationToken)
{
if (cancellationToken.IsCancellationRequested)
{
_logger.LogDebug($"Cannot retry {resourceEvent}, processing has been canceled");
return(false);
}
var upcoming = _eventManager.Peek(resourceEvent.ResourceUid);
if (upcoming != null)
{
_logger.LogDebug($"Cannot retry {resourceEvent}, received {upcoming} in the meantime");
return(false);
}
if (attemptNumber > RetryPolicy.MaxAttempts)
{
_logger.LogDebug($"Cannot retry {resourceEvent}, max number of attempts reached");
return(false);
}
return(true);
}
/// <summary>
/// Returns true if there is an event being handled
/// </summary>
private bool IsHandling(string resourceUid, out CustomResourceEvent handlingEvent)
{
return(_handling.TryGetValue(resourceUid, out handlingEvent));
}
