public async void WorkerError(WorkerErrorEvent workerError) { if (_disposing || _disposed) { return; } try { if (string.Equals(_workerRuntime, workerError.Language)) { _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception); AddOrUpdateErrorBucket(workerError); await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId, workerError.Exception); } else { _logger.LogDebug("Received WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId); _logger.LogDebug("WorkerErrorEvent runtime:{runtime} does not match current runtime:{currentRuntime}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception); } } catch (TaskCanceledException) { // Specifically in the "we were torn down while trying to restart" case, we want to catch here and ignore // If we don't catch the exception from an async void method, we'll end up tearing down the entire runtime instead // It's possible we want to catch *all* exceptions and log or ignore here, but taking the minimal change first // For example if we capture and log, we're left in a worker-less state with a working Host runtime - is that desired? Will it self recover elsewhere? } }
public void WorkerError(WorkerErrorEvent workerError) { ILanguageWorkerChannel erroredChannel; if (_channelsDictionary.TryGetValue(workerError.WorkerId, out erroredChannel)) { // TODO: move retry logic, possibly into worker channel decorator _channelState.AddOrUpdate(erroredChannel.Config, CreateWorkerState, (config, state) => { _erroredChannels.Add(state.Channel); state.Errors.Add(workerError.Exception); if (state.Errors.Count < 3) { state.Channel = _channelFactory(config, state.Functions); } else { var exception = new AggregateException(state.Errors.ToList()); var errorBlock = new ActionBlock <ScriptInvocationContext>(ctx => { ctx.ResultSource.TrySetException(exception); }); state.Functions.Subscribe(reg => reg.InputBuffer.LinkTo(errorBlock)); } return(state); }); } }
public async void WorkerError(WorkerErrorEvent workerError) { if (!_disposing) { _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId); AddOrUpdateErrorBucket(workerError); await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId); } }
public void WorkerError(WorkerErrorEvent workerError) { if (_workerStates.TryGetValue(workerError.Language, out LanguageWorkerState erroredWorkerState)) { erroredWorkerState.Errors.Add(workerError.Exception); bool isPreInitializedChannel = _languageWorkerChannelManager.ShutdownChannelIfExists(workerError.Language); if (!isPreInitializedChannel) { erroredWorkerState.Channel.Dispose(); } RestartWorkerChannel(workerError.Language, erroredWorkerState); } }
private void AddOrUpdateErrorBucket(WorkerErrorEvent currentErrorEvent) { if (_languageWorkerErrors.TryPeek(out WorkerErrorEvent top)) { if ((currentErrorEvent.CreatedAt - top.CreatedAt) > thresholdBetweenRestarts) { while (!_languageWorkerErrors.IsEmpty) { _languageWorkerErrors.TryPop(out WorkerErrorEvent popped); _logger.LogDebug($"Popping out errorEvent createdAt:{popped.CreatedAt} workerId:{popped.WorkerId}"); } } } _languageWorkerErrors.Push(currentErrorEvent); }
public void WorkerError(WorkerErrorEvent workerError) { if (_workerStates.TryGetValue(workerError.Language, out LanguageWorkerState erroredWorkerState)) { _logger.LogDebug($"Handling WorkerErrorEvent for runtime:{workerError.Language}"); erroredWorkerState.Errors.Add(workerError.Exception); bool isPreInitializedChannel = _languageWorkerChannelManager.ShutdownChannelIfExists(workerError.Language); if (!isPreInitializedChannel) { _logger.LogDebug($"Disposing errored channel for workerId: {0}, for runtime:{1}", erroredWorkerState.Channel.Id, workerError.Language); erroredWorkerState.Channel.Dispose(); } _logger.LogDebug($"Restarting worker channel for runtime:{0}", workerError.Language); RestartWorkerChannel(workerError.Language, erroredWorkerState); } }
public async void WorkerError(WorkerErrorEvent workerError) { if (!_disposing) { if (string.Equals(_workerRuntime, workerError.Language)) { _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId); AddOrUpdateErrorBucket(workerError); await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId); } else { _logger.LogDebug("Received WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId); _logger.LogDebug("WorkerErrorEvent runtime:{runtime} does not match current runtime:{currentRuntime}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception); } } }
public async void WorkerError(WorkerErrorEvent workerError) { if (!_disposing) { _logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId); _workerState.Errors.Add(workerError.Exception); bool isPreInitializedChannel = _languageWorkerChannelManager.ShutdownChannelIfExists(workerError.Language, workerError.WorkerId); if (!isPreInitializedChannel) { _logger.LogDebug("Disposing errored channel for workerId: {channelId}, for runtime:{language}", workerError.WorkerId, workerError.Language); var erroredChannel = _workerState.GetChannels().Where(ch => ch.Id == workerError.WorkerId).FirstOrDefault(); _workerState.DisposeAndRemoveChannel(erroredChannel); } _logger.LogDebug("Restarting worker channel for runtime:{runtime}", workerError.Language); await RestartWorkerChannel(workerError.Language, workerError.WorkerId); } }
public void WorkerError(WorkerErrorEvent workerError) { ILanguageWorkerChannel erroredChannel; if (_channelsDictionary.TryGetValue(workerError.WorkerId, out erroredChannel)) { // TODO: move retry logic, possibly into worker channel decorator _channelStates.AddOrUpdate(erroredChannel.Config, CreateWorkerState, (config, state) => { erroredChannel.Dispose(); state.Errors.Add(workerError.Exception); if (state.Errors.Count < 3) { state.Channel = _channelFactory(config, state.Functions, state.Errors.Count); _channelsDictionary[state.Channel.Id] = state.Channel; } else { var exMessage = $"Failed to start language worker for: {config.Language}"; var languageWorkerChannelException = (state.Errors != null && state.Errors.Count > 0) ? new LanguageWorkerChannelException(exMessage, new AggregateException(state.Errors.ToList())) : new LanguageWorkerChannelException(exMessage); var errorBlock = new ActionBlock <ScriptInvocationContext>(ctx => { ctx.ResultSource.TrySetException(languageWorkerChannelException); }); _workerStateSubscriptions.Add(state.Functions.Subscribe(reg => { state.AddRegistration(reg); reg.InputBuffer.LinkTo(errorBlock); })); _eventManager.Publish(new WorkerProcessErrorEvent(state.Channel.Id, config.Language, languageWorkerChannelException)); } return(state); }); } }