private async Task <bool> TryHandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken)
        {
            bool handled = true;

            try
            {
                var resource = (T)resourceEvent.Resource;

                if (IsDeletePending(resource))
                {
                    await HandleDeletedEventAsync(resource, cancellationToken);
                }
                else
                {
                    await HandleAddedOrModifiedEventAsync(resource, cancellationToken);
                }
            }
            catch (OperationCanceledException)
            {
                _logger.LogDebug($"Canceled HandleEvent, {resourceEvent}");
            }
            catch (Exception exception)
            {
                if (exception is HttpOperationException httpException && httpException.Response?.StatusCode == System.Net.HttpStatusCode.Conflict)
                {
                    // Conflicts happen. The next event will make the resource consistent again
                    _logger.LogDebug(exception, $"Conflict handling {resourceEvent}");
                }
 /// <summary>
 /// Track the end of an event handling
 /// </summary>
 public void EndHandleEvent(CustomResourceEvent resourceEvent)
 {
     lock (this)
     {
         _logger.LogTrace($"EndHandleEvent {resourceEvent}");
         _handling.Remove(resourceEvent.ResourceUid);
     }
 }
 /// <summary>
 /// Track the begin of an event handling
 /// </summary>
 public void BeginHandleEvent(CustomResourceEvent resourceEvent)
 {
     lock (this)
     {
         _logger.LogTrace($"BeginHandleEvent {resourceEvent}");
         _handling[resourceEvent.ResourceUid] = resourceEvent;
     }
 }
 /// <summary>
 /// Enqueue the event
 /// </summary>
 public void Enqueue(CustomResourceEvent resourceEvent)
 {
     lock (this)
     {
         _logger.LogTrace($"Enqueue {resourceEvent}");
         // Insert or update the next event for the resource
         _queuesByResource[resourceEvent.ResourceUid] = resourceEvent;
     }
 }
        /// <summary>
        /// Dispatches an incoming event to the controller
        /// </summary>
        public void OnIncomingEvent(WatchEventType eventType, CustomResource resource)
        {
            var resourceEvent = new CustomResourceEvent(eventType, resource);

            _logger.LogDebug($"Received event {resourceEvent}");

            Controller.ProcessEventAsync(resourceEvent, _cancellationToken)
            .ContinueWith(t =>
            {
                if (t.IsFaulted)
                {
                    var exception = t.Exception.Flatten().InnerException;
                    _logger.LogError(exception, $"Error processing {resourceEvent}");
                }
            });
        }
        private async Task HandleEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken)
        {
            if (resourceEvent == null)
            {
                _logger.LogWarning($"Skip HandleEvent, {nameof(resourceEvent)} is null");
                return;
            }

            _logger.LogDebug($"Begin HandleEvent, {resourceEvent}");

            _eventManager.BeginHandleEvent(resourceEvent);

            var attempt = 1;
            var delay   = RetryPolicy.InitialDelay;

            while (true)
            {
                // Try to handle the event
                var handled = await TryHandleEventAsync(resourceEvent, cancellationToken);

                if (handled)
                {
                    break;
                }

                // Something went wrong
                if (!CanTryAgain(resourceEvent, attempt, cancellationToken))
                {
                    break;
                }

                _logger.LogDebug($"Retrying to handle {resourceEvent} in {delay}ms (attempt #{attempt})");

                // Wait
                await Task.Delay(delay);

                // Increase the delay for the next attempt
                attempt++;
                delay = (int)(delay * RetryPolicy.DelayMultiplier);
            }

            _logger.LogDebug($"End HandleEvent, {resourceEvent}");

            _eventManager.EndHandleEvent(resourceEvent);
        }
        /// <summary>
        /// Processes a custom resource event
        /// </summary>
        /// <param name="resourceEvent">The event to handle</param>
        /// <param name="cancellationToken">Signals if the current execution has been canceled</param>
        public async Task ProcessEventAsync(CustomResourceEvent resourceEvent, CancellationToken cancellationToken)
        {
            _logger.LogDebug($"Begin ProcessEvent, {resourceEvent}");

            if (resourceEvent.Type == WatchEventType.Error)
            {
                _logger.LogError($"Received Error event, {resourceEvent.Resource}");
                return;
            }

            if (resourceEvent.Type == WatchEventType.Deleted)
            {
                // Skip Deleted events since there is nothing else to do
                _logger.LogDebug($"Skip ProcessEvent, received Deleted event, {resourceEvent.Resource}");
                return;
            }

            if (resourceEvent.Type == WatchEventType.Bookmark)
            {
                // Skip Bookmark events since there is nothing else to do
                _logger.LogDebug($"Skip ProcessEvent, received Bookmark event, {resourceEvent.Resource}");
                return;
            }

            // Enqueue the event
            _eventManager.Enqueue(resourceEvent);

            while (!cancellationToken.IsCancellationRequested)
            {
                // Dequeue the next event to process for this resource, if any
                var nextEvent = _eventManager.Dequeue(resourceEvent.ResourceUid);
                if (nextEvent == null)
                {
                    break;
                }

                await HandleEventAsync(nextEvent, cancellationToken);
            }

            _logger.LogDebug($"End ProcessEvent, {resourceEvent}");
        }
        private bool CanTryAgain(CustomResourceEvent resourceEvent, int attemptNumber, CancellationToken cancellationToken)
        {
            if (cancellationToken.IsCancellationRequested)
            {
                _logger.LogDebug($"Cannot retry {resourceEvent}, processing has been canceled");
                return(false);
            }

            var upcoming = _eventManager.Peek(resourceEvent.ResourceUid);

            if (upcoming != null)
            {
                _logger.LogDebug($"Cannot retry {resourceEvent}, received {upcoming} in the meantime");
                return(false);
            }

            if (attemptNumber > RetryPolicy.MaxAttempts)
            {
                _logger.LogDebug($"Cannot retry {resourceEvent}, max number of attempts reached");
                return(false);
            }

            return(true);
        }
 /// <summary>
 /// Returns true if there is an event being handled
 /// </summary>
 private bool IsHandling(string resourceUid, out CustomResourceEvent handlingEvent)
 {
     return(_handling.TryGetValue(resourceUid, out handlingEvent));
 }