private async Task <bool> TryHandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken) { bool handled = true; try { var resource = (T)resourceEvent.Resource; if (IsDeletePending(resource)) { await HandleDeletedEventAsync(resource, cancellationToken); } else { await HandleAddedOrModifiedEventAsync(resource, cancellationToken); } } catch (OperationCanceledException) { _logger.LogDebug($"Canceled HandleEvent, {resourceEvent}"); } catch (Exception exception) { if (exception is HttpOperationException httpException && httpException.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) { // Conflicts happen. The next event will make the resource consistent again _logger.LogDebug(exception, $"Conflict handling {resourceEvent}"); }
/// <summary> /// Track the end of an event handling /// </summary> public void EndHandleEvent(CustomResourceEvent resourceEvent) { lock (this) { _logger.LogTrace($"EndHandleEvent {resourceEvent}"); _handling.Remove(resourceEvent.ResourceUid); } }
/// <summary> /// Track the begin of an event handling /// </summary> public void BeginHandleEvent(CustomResourceEvent resourceEvent) { lock (this) { _logger.LogTrace($"BeginHandleEvent {resourceEvent}"); _handling[resourceEvent.ResourceUid] = resourceEvent; } }
/// <summary> /// Enqueue the event /// </summary> public void Enqueue(CustomResourceEvent resourceEvent) { lock (this) { _logger.LogTrace($"Enqueue {resourceEvent}"); // Insert or update the next event for the resource _queuesByResource[resourceEvent.ResourceUid] = resourceEvent; } }
/// <summary> /// Dispatches an incoming event to the controller /// </summary> public void OnIncomingEvent(WatchEventType eventType, CustomResource resource) { var resourceEvent = new CustomResourceEvent(eventType, resource); _logger.LogDebug($"Received event {resourceEvent}"); Controller.ProcessEventAsync(resourceEvent, _cancellationToken) .ContinueWith(t => { if (t.IsFaulted) { var exception = t.Exception.Flatten().InnerException; _logger.LogError(exception, $"Error processing {resourceEvent}"); } }); }
private async Task HandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken) { if (resourceEvent == null) { _logger.LogWarning($"Skip HandleEvent, {nameof(resourceEvent)} is null"); return; } _logger.LogDebug($"Begin HandleEvent, {resourceEvent}"); _eventManager.BeginHandleEvent(resourceEvent); var attempt = 1; var delay = RetryPolicy.InitialDelay; while (true) { // Try to handle the event var handled = await TryHandleEventAsync(resourceEvent, cancellationToken); if (handled) { break; } // Something went wrong if (!CanTryAgain(resourceEvent, attempt, cancellationToken)) { break; } _logger.LogDebug($"Retrying to handle {resourceEvent} in {delay}ms (attempt #{attempt})"); // Wait await Task.Delay(delay); // Increase the delay for the next attempt attempt++; delay = (int)(delay * RetryPolicy.DelayMultiplier); } _logger.LogDebug($"End HandleEvent, {resourceEvent}"); _eventManager.EndHandleEvent(resourceEvent); }
/// <summary> /// Processes a custom resource event /// </summary> /// <param name="resourceEvent">The event to handle</param> /// <param name="cancellationToken">Signals if the current execution has been canceled</param> public async Task ProcessEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken) { _logger.LogDebug($"Begin ProcessEvent, {resourceEvent}"); if (resourceEvent.Type == WatchEventType.Error) { _logger.LogError($"Received Error event, {resourceEvent.Resource}"); return; } if (resourceEvent.Type == WatchEventType.Deleted) { // Skip Deleted events since there is nothing else to do _logger.LogDebug($"Skip ProcessEvent, received Deleted event, {resourceEvent.Resource}"); return; } if (resourceEvent.Type == WatchEventType.Bookmark) { // Skip Bookmark events since there is nothing else to do _logger.LogDebug($"Skip ProcessEvent, received Bookmark event, {resourceEvent.Resource}"); return; } // Enqueue the event _eventManager.Enqueue(resourceEvent); while (!cancellationToken.IsCancellationRequested) { // Dequeue the next event to process for this resource, if any var nextEvent = _eventManager.Dequeue(resourceEvent.ResourceUid); if (nextEvent == null) { break; } await HandleEventAsync(nextEvent, cancellationToken); } _logger.LogDebug($"End ProcessEvent, {resourceEvent}"); }
private bool CanTryAgain(CustomResourceEvent resourceEvent, int attemptNumber, CancellationToken cancellationToken) { if (cancellationToken.IsCancellationRequested) { _logger.LogDebug($"Cannot retry {resourceEvent}, processing has been canceled"); return(false); } var upcoming = _eventManager.Peek(resourceEvent.ResourceUid); if (upcoming != null) { _logger.LogDebug($"Cannot retry {resourceEvent}, received {upcoming} in the meantime"); return(false); } if (attemptNumber > RetryPolicy.MaxAttempts) { _logger.LogDebug($"Cannot retry {resourceEvent}, max number of attempts reached"); return(false); } return(true); }
/// <summary> /// Returns true if there is an event being handled /// </summary> private bool IsHandling(string resourceUid, out CustomResourceEvent handlingEvent) { return(_handling.TryGetValue(resourceUid, out handlingEvent)); }