public async void WorkerError(WorkerErrorEvent workerError)
{
if (_disposing || _disposed)
{
return;
}
try
{
if (string.Equals(_workerRuntime, workerError.Language))
{
_logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception);
AddOrUpdateErrorBucket(workerError);
await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId, workerError.Exception);
}
else
{
_logger.LogDebug("Received WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
_logger.LogDebug("WorkerErrorEvent runtime:{runtime} does not match current runtime:{currentRuntime}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception);
}
}
catch (TaskCanceledException)
{
// Specifically in the "we were torn down while trying to restart" case, we want to catch here and ignore
// If we don't catch the exception from an async void method, we'll end up tearing down the entire runtime instead
// It's possible we want to catch *all* exceptions and log or ignore here, but taking the minimal change first
// For example if we capture and log, we're left in a worker-less state with a working Host runtime - is that desired? Will it self recover elsewhere?
}
}
public void WorkerError(WorkerErrorEvent workerError)
{
ILanguageWorkerChannel erroredChannel;
if (_channelsDictionary.TryGetValue(workerError.WorkerId, out erroredChannel))
{
// TODO: move retry logic, possibly into worker channel decorator
_channelState.AddOrUpdate(erroredChannel.Config,
CreateWorkerState,
(config, state) =>
{
_erroredChannels.Add(state.Channel);
state.Errors.Add(workerError.Exception);
if (state.Errors.Count < 3)
{
state.Channel = _channelFactory(config, state.Functions);
}
else
{
var exception = new AggregateException(state.Errors.ToList());
var errorBlock = new ActionBlock <ScriptInvocationContext>(ctx =>
{
ctx.ResultSource.TrySetException(exception);
});
state.Functions.Subscribe(reg => reg.InputBuffer.LinkTo(errorBlock));
}
return(state);
});
}
}
public async void WorkerError(WorkerErrorEvent workerError)
{
if (!_disposing)
{
_logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
AddOrUpdateErrorBucket(workerError);
await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId);
}
}
public void WorkerError(WorkerErrorEvent workerError)
{
if (_workerStates.TryGetValue(workerError.Language, out LanguageWorkerState erroredWorkerState))
{
erroredWorkerState.Errors.Add(workerError.Exception);
bool isPreInitializedChannel = _languageWorkerChannelManager.ShutdownChannelIfExists(workerError.Language);
if (!isPreInitializedChannel)
{
erroredWorkerState.Channel.Dispose();
}
RestartWorkerChannel(workerError.Language, erroredWorkerState);
}
}
private void AddOrUpdateErrorBucket(WorkerErrorEvent currentErrorEvent)
{
if (_languageWorkerErrors.TryPeek(out WorkerErrorEvent top))
{
if ((currentErrorEvent.CreatedAt - top.CreatedAt) > thresholdBetweenRestarts)
{
while (!_languageWorkerErrors.IsEmpty)
{
_languageWorkerErrors.TryPop(out WorkerErrorEvent popped);
_logger.LogDebug($"Popping out errorEvent createdAt:{popped.CreatedAt} workerId:{popped.WorkerId}");
}
}
}
_languageWorkerErrors.Push(currentErrorEvent);
}
public void WorkerError(WorkerErrorEvent workerError)
{
if (_workerStates.TryGetValue(workerError.Language, out LanguageWorkerState erroredWorkerState))
{
_logger.LogDebug($"Handling WorkerErrorEvent for runtime:{workerError.Language}");
erroredWorkerState.Errors.Add(workerError.Exception);
bool isPreInitializedChannel = _languageWorkerChannelManager.ShutdownChannelIfExists(workerError.Language);
if (!isPreInitializedChannel)
{
_logger.LogDebug($"Disposing errored channel for workerId: {0}, for runtime:{1}", erroredWorkerState.Channel.Id, workerError.Language);
erroredWorkerState.Channel.Dispose();
}
_logger.LogDebug($"Restarting worker channel for runtime:{0}", workerError.Language);
RestartWorkerChannel(workerError.Language, erroredWorkerState);
}
}
public async void WorkerError(WorkerErrorEvent workerError)
{
if (!_disposing)
{
if (string.Equals(_workerRuntime, workerError.Language))
{
_logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
AddOrUpdateErrorBucket(workerError);
await DisposeAndRestartWorkerChannel(workerError.Language, workerError.WorkerId);
}
else
{
_logger.LogDebug("Received WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
_logger.LogDebug("WorkerErrorEvent runtime:{runtime} does not match current runtime:{currentRuntime}. Failed with: {exception}", workerError.Language, _workerRuntime, workerError.Exception);
}
}
}
public async void WorkerError(WorkerErrorEvent workerError)
{
if (!_disposing)
{
_logger.LogDebug("Handling WorkerErrorEvent for runtime:{runtime}, workerId:{workerId}", workerError.Language, workerError.WorkerId);
_workerState.Errors.Add(workerError.Exception);
bool isPreInitializedChannel = _languageWorkerChannelManager.ShutdownChannelIfExists(workerError.Language, workerError.WorkerId);
if (!isPreInitializedChannel)
{
_logger.LogDebug("Disposing errored channel for workerId: {channelId}, for runtime:{language}", workerError.WorkerId, workerError.Language);
var erroredChannel = _workerState.GetChannels().Where(ch => ch.Id == workerError.WorkerId).FirstOrDefault();
_workerState.DisposeAndRemoveChannel(erroredChannel);
}
_logger.LogDebug("Restarting worker channel for runtime:{runtime}", workerError.Language);
await RestartWorkerChannel(workerError.Language, workerError.WorkerId);
}
}
public void WorkerError(WorkerErrorEvent workerError)
{
ILanguageWorkerChannel erroredChannel;
if (_channelsDictionary.TryGetValue(workerError.WorkerId, out erroredChannel))
{
// TODO: move retry logic, possibly into worker channel decorator
_channelStates.AddOrUpdate(erroredChannel.Config,
CreateWorkerState,
(config, state) =>
{
erroredChannel.Dispose();
state.Errors.Add(workerError.Exception);
if (state.Errors.Count < 3)
{
state.Channel = _channelFactory(config, state.Functions, state.Errors.Count);
_channelsDictionary[state.Channel.Id] = state.Channel;
}
else
{
var exMessage = $"Failed to start language worker for: {config.Language}";
var languageWorkerChannelException = (state.Errors != null && state.Errors.Count > 0) ? new LanguageWorkerChannelException(exMessage, new AggregateException(state.Errors.ToList())) : new LanguageWorkerChannelException(exMessage);
var errorBlock = new ActionBlock <ScriptInvocationContext>(ctx =>
{
ctx.ResultSource.TrySetException(languageWorkerChannelException);
});
_workerStateSubscriptions.Add(state.Functions.Subscribe(reg =>
{
state.AddRegistration(reg);
reg.InputBuffer.LinkTo(errorBlock);
}));
_eventManager.Publish(new WorkerProcessErrorEvent(state.Channel.Id, config.Language, languageWorkerChannelException));
}
return(state);
});
}
}
