/// <summary>
/// Whenever the debug sentinel file changes we update our debug timeout
/// </summary>
private void OnDebugModeFileChanged(object sender, FileSystemEventArgs e)
{
if (!_disposed)
{
_eventManager.Publish(new DebugNotification(nameof(FileMonitoringService), DateTime.UtcNow));
}
}
internal void WorkerReady(RpcEvent initEvent)
{
_startLatencyMetric.Dispose();
_startLatencyMetric = null;
var initMessage = initEvent.Message.WorkerInitResponse;
if (initMessage.Result.IsFailure(out Exception exc))
{
HandleWorkerError(exc);
return;
}
// subscript to all function registrations in order to load functions
_eventSubscriptions.Add(_functionRegistrations.Subscribe(Register));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.FunctionLoadResponse)
.Subscribe((msg) => LoadResponse(msg.Message.FunctionLoadResponse)));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.InvocationResponse)
.Subscribe((msg) => InvokeResponse(msg.Message.InvocationResponse)));
_eventManager.Publish(new WorkerReadyEvent
{
Id = _workerId,
Version = initMessage.WorkerVersion,
Capabilities = initMessage.Capabilities,
Config = _workerConfig,
});
}
internal void HandleWorkerChannelError(Exception exc)
{
if (_disposing)
{
return;
}
_eventManager.Publish(new WorkerErrorEvent(_runtime, Id, exc));
}
internal override void HandleWorkerProcessExitError(WorkerProcessExitException langExc)
{
// The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel
if (langExc != null && langExc.ExitCode != -1)
{
_workerProcessLogger.LogDebug(langExc, $"Language Worker Process exited.", _workerProcessArguments.ExecutablePath);
_eventManager.Publish(new HttpWorkerErrorEvent(_workerId, langExc));
}
}
internal void HandleWorkerProcessExitError(LanguageWorkerProcessExitException langExc)
{
// The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel
if (langExc != null && langExc.ExitCode != -1)
{
_workerProcessLogger.LogDebug(langExc, $"Language Worker Process exited.", _process.StartInfo.FileName);
_eventManager.Publish(new WorkerErrorEvent(_runtime, _workerId, langExc));
}
}
internal void HandleWorkerFunctionLoadError(Exception exc)
{
_workerChannelLogger.LogError(exc, "Loading function failed.");
if (_disposing || _disposed)
{
return;
}
_eventManager.Publish(new WorkerErrorEvent(_runtime, Id, exc));
}
private void PublishWorkerErrorEvent(Exception exc)
{
_workerInitTask.SetException(exc);
if (_disposing)
{
return;
}
_eventManager.Publish(new WorkerErrorEvent(_runtime, Id, exc));
}
internal override void HandleWorkerProcessExitError(WorkerProcessExitException rpcWorkerProcessExitException)
{
if (rpcWorkerProcessExitException == null)
{
throw new ArgumentNullException(nameof(rpcWorkerProcessExitException));
}
// The subscriber of WorkerErrorEvent is expected to Dispose() the errored channel
_workerProcessLogger.LogDebug(rpcWorkerProcessExitException, $"Language Worker Process exited. Pid={rpcWorkerProcessExitException.Pid}.", _workerProcessArguments.ExecutablePath);
_eventManager.Publish(new WorkerErrorEvent(_runtime, _workerId, rpcWorkerProcessExitException));
}
internal void PublishWorkerProcessReadyEvent(FunctionEnvironmentReloadResponse res)
{
if (_disposing)
{
// do not publish ready events when disposing
return;
}
WorkerProcessReadyEvent wpEvent = new WorkerProcessReadyEvent(_workerId, _workerConfig.Language);
_eventManager.Publish(wpEvent);
}
public override async Task EventStream(IAsyncStreamReader <StreamingMessage> requestStream, IServerStreamWriter <StreamingMessage> responseStream, ServerCallContext context)
{
var cancelSource = new TaskCompletionSource <bool>();
IDisposable outboundEventSubscription = null;
try
{
context.CancellationToken.Register(() => cancelSource.TrySetResult(false));
Func <Task <bool> > messageAvailable = async() =>
{
// GRPC does not accept cancellation tokens for individual reads, hence wrapper
var requestTask = requestStream.MoveNext(CancellationToken.None);
var completed = await Task.WhenAny(cancelSource.Task, requestTask);
return(completed.Result);
};
if (await messageAvailable())
{
string workerId = requestStream.Current.StartStream.WorkerId;
outboundEventSubscription = _eventManager.OfType <OutboundEvent>()
.Where(evt => evt.WorkerId == workerId)
.ObserveOn(NewThreadScheduler.Default)
.Subscribe(evt =>
{
try
{
// WriteAsync only allows one pending write at a time
// For each responseStream subscription, observe as a blocking write, in series, on a new thread
// Alternatives - could wrap responseStream.WriteAsync with a SemaphoreSlim to control concurrent access
responseStream.WriteAsync(evt.Message).GetAwaiter().GetResult();
}
catch (Exception subscribeEventEx)
{
_eventManager.Publish(new WorkerErrorEvent(workerId, subscribeEventEx));
}
});
do
{
_eventManager.Publish(new InboundEvent(workerId, requestStream.Current));
}while (await messageAvailable());
}
}
finally
{
outboundEventSubscription?.Dispose();
// ensure cancellationSource task completes
cancelSource.TrySetResult(false);
}
}
internal void InvocationRequest(StreamingMessage serverMessage)
{
InvocationRequest invocationRequest = serverMessage.InvocationRequest;
InvocationResponse invocationResponse = new InvocationResponse()
{
InvocationId = invocationRequest.InvocationId,
Result = "Success"
};
StreamingMessage responseMessage = new StreamingMessage()
{
InvocationResponse = invocationResponse
};
_eventManager.Publish(new OutboundEvent(_workerId, responseMessage));
}
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
// propagate special exceptions through the EventManager
var stateProps = state as IEnumerable <KeyValuePair <string, object> > ?? new Dictionary <string, object>();
string source = _categoryName ?? Utility.GetStateValueOrDefault <string>(stateProps, ScriptConstants.LogPropertySourceKey);
if (exception is FunctionIndexingException && _eventManager != null)
{
_eventManager.Publish(new FunctionIndexingEvent("FunctionIndexingException", source, exception));
}
// User logs are not logged to system logs.
if (!IsEnabled(logLevel) || IsUserLog(state))
{
return;
}
string formattedMessage = formatter?.Invoke(state, exception);
// If we don't have a message, there's nothing to log.
if (string.IsNullOrEmpty(formattedMessage))
{
return;
}
IDictionary <string, object> scopeProps = _scopeProvider.GetScopeDictionary();
// Apply standard event properties
// Note: we must be sure to default any null values to empty string
// otherwise the ETW event will fail to be persisted (silently)
string subscriptionId = _environment.GetSubscriptionId() ?? string.Empty;
string appName = _environment.GetAzureWebsiteUniqueSlotName() ?? string.Empty;
string summary = Sanitizer.Sanitize(formattedMessage) ?? string.Empty;
string innerExceptionType = string.Empty;
string innerExceptionMessage = string.Empty;
string functionName = _functionName ?? Utility.ResolveFunctionName(stateProps, scopeProps) ?? string.Empty;
string eventName = !string.IsNullOrEmpty(eventId.Name) ? eventId.Name : Utility.GetStateValueOrDefault <string>(stateProps, ScriptConstants.LogPropertyEventNameKey) ?? string.Empty;
string functionInvocationId = Utility.GetValueFromScope(scopeProps, ScriptConstants.LogPropertyFunctionInvocationIdKey) ?? string.Empty;
string hostInstanceId = _hostInstanceId;
string activityId = Utility.GetStateValueOrDefault <string>(stateProps, ScriptConstants.LogPropertyActivityIdKey) ?? string.Empty;
string runtimeSiteName = _environment.GetRuntimeSiteName() ?? string.Empty;
string slotName = _environment.GetSlotName() ?? string.Empty;
// Populate details from the exception.
string details = string.Empty;
if (exception != null)
{
if (string.IsNullOrEmpty(functionName) && exception is FunctionInvocationException fex)
{
functionName = string.IsNullOrEmpty(fex.MethodName) ? string.Empty : fex.MethodName.Replace("Host.Functions.", string.Empty);
}
(innerExceptionType, innerExceptionMessage, details) = exception.GetExceptionDetails();
innerExceptionMessage = innerExceptionMessage ?? string.Empty;
}
_eventGenerator.LogFunctionTraceEvent(logLevel, subscriptionId, appName, functionName, eventName, source, details, summary, innerExceptionType, innerExceptionMessage, functionInvocationId, hostInstanceId, activityId, runtimeSiteName, slotName);
}
private void PublishWorkerErrorEvent(Exception exc)
{
if (_disposed)
{
return;
}
_eventManager.Publish(new HttpWorkerErrorEvent(Id, exc));
}
public async Task RpcStreamReader()
{
while (await _call.ResponseStream.MoveNext())
{
var serverMessage = _call.ResponseStream.Current;
_eventManager.Publish(new InboundEvent(_workerId, serverMessage));
}
}
private void FileChanged(object sender, FileSystemEventArgs e)
{
if (!_disposed)
{
var fileEvent = new FileEvent(_source, e);
_eventManager.Publish(fileEvent);
}
}
public void PublishFunctionLoadResponseEvent(string functionId)
{
StatusResult statusResult = new StatusResult()
{
Status = StatusResult.Types.Status.Success
};
FunctionLoadResponse functionLoadResponse = new FunctionLoadResponse()
{
FunctionId = functionId,
Result = statusResult
};
StreamingMessage responseMessage = new StreamingMessage()
{
FunctionLoadResponse = functionLoadResponse
};
_eventManager.Publish(new InboundEvent(_workerId, responseMessage));
}
private ILanguageWorkerChannel CreateTestChannel(string workerId, string language)
{
var testChannel = _languageWorkerChannelManager.CreateLanguageWorkerChannel(workerId, _scriptRootPath, language, null, null, 0);
// Generate event to mock language worker response
RpcChannelReadyEvent javarReadyEvent = new RpcChannelReadyEvent(workerId, language, testChannel, "testVersion", _capabilities);
_eventManager.Publish(javarReadyEvent);
return(testChannel);
}
internal GrpcWorkerChannel(
string workerId,
IScriptEventManager eventManager,
RpcWorkerConfig workerConfig,
IWorkerProcess rpcWorkerProcess,
ILogger logger,
IMetricsLogger metricsLogger,
int attemptCount,
IEnvironment environment,
IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions,
ISharedMemoryManager sharedMemoryManager,
IFunctionDataCache functionDataCache,
IOptions <WorkerConcurrencyOptions> workerConcurrencyOptions)
{
_workerId = workerId;
_eventManager = eventManager;
_workerConfig = workerConfig;
_runtime = workerConfig.Description.Language;
_rpcWorkerProcess = rpcWorkerProcess;
_workerChannelLogger = logger;
_metricsLogger = metricsLogger;
_environment = environment;
_applicationHostOptions = applicationHostOptions;
_sharedMemoryManager = sharedMemoryManager;
_workerConcurrencyOptions = workerConcurrencyOptions;
_workerCapabilities = new GrpcCapabilities(_workerChannelLogger);
_inboundWorkerEvents = _eventManager.OfType <InboundGrpcEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.IsMessageOfType(MsgType.RpcLog) && !msg.IsLogOfCategory(RpcLogCategory.System))
.Subscribe(Log));
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.IsMessageOfType(MsgType.RpcLog) && msg.IsLogOfCategory(RpcLogCategory.System))
.Subscribe(SystemLog));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => _workerConfig.Description.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.InvocationResponse)
.Subscribe(async(msg) => await InvokeResponse(msg.Message.InvocationResponse)));
_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.WorkerStatusResponse)
.Subscribe((msg) => ReceiveWorkerStatusResponse(msg.Message.RequestId, msg.Message.WorkerStatusResponse));
_startLatencyMetric = metricsLogger?.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Description.Language, attemptCount));
_state = RpcWorkerChannelState.Default;
}
public async void Publish(ScriptEvent scriptEvent)
{
// Emulate long worker status latency
await Task.Delay(WaitBeforePublish);
try
{
_scriptEventManager.Publish(scriptEvent);
}
catch (ObjectDisposedException)
{
// Do no throw ObjectDisposedException
}
}
public Task StartWorkerProcessAsync()
{
// To verify FunctionDispatcher transistions
Task.Delay(TimeSpan.FromMilliseconds(100));
string workerVersion = Guid.NewGuid().ToString();
IDictionary <string, string> workerCapabilities = new Dictionary <string, string>()
{
{ "test", "testSupported" }
};
if (_isWebhostChannel)
{
RpcWebHostChannelReadyEvent readyEvent = new RpcWebHostChannelReadyEvent(_workerId, _runtime, this, workerVersion, workerCapabilities);
_eventManager.Publish(readyEvent);
}
else
{
RpcJobHostChannelReadyEvent readyEvent = new RpcJobHostChannelReadyEvent(_workerId, _runtime, this, workerVersion, workerCapabilities);
_eventManager.Publish(readyEvent);
}
_state = LanguageWorkerChannelState.Initialized;
return(Task.CompletedTask);
}
public LanguageWorkerChannel(
ScriptJobHostOptions scriptConfig,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
IObservable <FunctionRegistrationContext> functionRegistrations,
WorkerConfig workerConfig,
Uri serverUri,
ILoggerFactory loggerFactory,
IMetricsLogger metricsLogger,
int attemptCount)
{
_workerId = Guid.NewGuid().ToString();
_scriptConfig = scriptConfig;
_eventManager = eventManager;
_processFactory = processFactory;
_processRegistry = processRegistry;
_functionRegistrations = functionRegistrations;
_workerConfig = workerConfig;
_serverUri = serverUri;
_workerChannelLogger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}");
_userLogsConsoleLogger = loggerFactory.CreateLogger(LanguageWorkerConstants.FunctionConsoleLogCategoryName);
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.MessageType == MsgType.RpcLog)
.Subscribe(Log));
_eventSubscriptions.Add(_eventManager.OfType <RpcEvent>()
.Where(msg => msg.WorkerId == _workerId)
.Subscribe(msg =>
{
var jsonMsg = JsonConvert.SerializeObject(msg, _verboseSerializerSettings);
_userLogsConsoleLogger.LogTrace(jsonMsg);
}));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => Config.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_startLatencyMetric = metricsLogger.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Language, attemptCount));
StartWorker();
}
internal RpcWorkerChannel(
string workerId,
string rootScriptPath,
IScriptEventManager eventManager,
RpcWorkerConfig workerConfig,
IWorkerProcess rpcWorkerProcess,
ILogger logger,
IMetricsLogger metricsLogger,
int attemptCount,
IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
{
_workerId = workerId;
_rootScriptPath = rootScriptPath;
_eventManager = eventManager;
_workerConfig = workerConfig;
_runtime = workerConfig.Description.Language;
_rpcWorkerProcess = rpcWorkerProcess;
_workerChannelLogger = logger;
_metricsLogger = metricsLogger;
_workerCapabilities = new Capabilities(_workerChannelLogger);
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.IsMessageOfType(MsgType.RpcLog) && !msg.IsLogOfCategory(RpcLogCategory.System))
.Subscribe(Log));
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.IsMessageOfType(MsgType.RpcLog) && msg.IsLogOfCategory(RpcLogCategory.System))
.Subscribe(SystemLog));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => _workerConfig.Description.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.FunctionLoadResponse)
.Subscribe((msg) => LoadResponse(msg.Message.FunctionLoadResponse)));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.InvocationResponse)
.Subscribe((msg) => InvokeResponse(msg.Message.InvocationResponse)));
_startLatencyMetric = metricsLogger?.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Description.Language, attemptCount));
_managedDependencyOptions = managedDependencyOptions;
_state = RpcWorkerChannelState.Default;
}
internal LanguageWorkerChannel(
string workerId,
string rootScriptPath,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
WorkerConfig workerConfig,
Uri serverUri,
ILoggerFactory loggerFactory,
IMetricsLogger metricsLogger,
int attemptCount,
ILanguageWorkerConsoleLogSource consoleLogSource,
bool isWebHostChannel = false,
IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
{
_workerId = workerId;
_rootScriptPath = rootScriptPath;
_eventManager = eventManager;
_processFactory = processFactory;
_processRegistry = processRegistry;
_workerConfig = workerConfig;
_serverUri = serverUri;
_workerChannelLogger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}");
_consoleLogSource = consoleLogSource;
_isWebHostChannel = isWebHostChannel;
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.MessageType == MsgType.RpcLog)
.Subscribe(Log));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => Config.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.FunctionLoadResponse)
.Subscribe((msg) => LoadResponse(msg.Message.FunctionLoadResponse)));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.InvocationResponse)
.Subscribe((msg) => InvokeResponse(msg.Message.InvocationResponse)));
_startLatencyMetric = metricsLogger?.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Language, attemptCount));
_managedDependencyOptions = managedDependencyOptions;
_state = LanguageWorkerChannelState.Default;
}
public LanguageWorkerChannel(
ScriptHostConfiguration scriptConfig,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
IObservable <FunctionRegistrationContext> functionRegistrations,
WorkerConfig workerConfig,
Uri serverUri,
ILoggerFactory loggerFactory)
{
_workerId = Guid.NewGuid().ToString();
_scriptConfig = scriptConfig;
_eventManager = eventManager;
_processFactory = processFactory;
_processRegistry = processRegistry;
_functionRegistrations = functionRegistrations;
_workerConfig = workerConfig;
_serverUri = serverUri;
_logger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}");
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.MessageType == MsgType.RpcLog)
.Subscribe(Log));
if (scriptConfig.LogFilter.Filter("Worker", LogLevel.Trace))
{
_eventSubscriptions.Add(_eventManager.OfType <RpcEvent>()
.Where(msg => msg.WorkerId == _workerId)
.Subscribe(msg =>
{
var jsonMsg = JsonConvert.SerializeObject(msg, _verboseSerializerSettings);
// TODO: change to trace when ILogger & TraceWriter merge (issues with file trace writer)
_logger.LogInformation(jsonMsg);
}));
}
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => Path.GetExtension(msg.FileChangeArguments.FullPath) == Config.Extension)
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
StartWorker();
}
private void PublishWorkerProcessErrorEvent(string language, LanguageWorkerState erroredWorkerState)
{
var exMessage = $"Failed to start language worker for: {language}";
var languageWorkerChannelException = (erroredWorkerState.Errors != null && erroredWorkerState.Errors.Count > 0) ? new LanguageWorkerChannelException(exMessage, new AggregateException(erroredWorkerState.Errors.ToList())) : new LanguageWorkerChannelException(exMessage);
var errorBlock = new ActionBlock <ScriptInvocationContext>(ctx =>
{
ctx.ResultSource.TrySetException(languageWorkerChannelException);
});
_workerStateSubscriptions.Add(erroredWorkerState.Functions.Subscribe(reg =>
{
erroredWorkerState.AddRegistration(reg);
reg.InputBuffer.LinkTo(errorBlock);
}));
_eventManager.Publish(new WorkerProcessErrorEvent(erroredWorkerState.Channel.Id, language, languageWorkerChannelException));
}
public void WorkerError(WorkerErrorEvent workerError)
{
ILanguageWorkerChannel erroredChannel;
if (_channelsDictionary.TryGetValue(workerError.WorkerId, out erroredChannel))
{
// TODO: move retry logic, possibly into worker channel decorator
_channelStates.AddOrUpdate(erroredChannel.Config,
CreateWorkerState,
(config, state) =>
{
erroredChannel.Dispose();
state.Errors.Add(workerError.Exception);
if (state.Errors.Count < 3)
{
state.Channel = _channelFactory(config, state.Functions, state.Errors.Count);
_channelsDictionary[state.Channel.Id] = state.Channel;
}
else
{
var exMessage = $"Failed to start language worker for: {config.Language}";
var languageWorkerChannelException = (state.Errors != null && state.Errors.Count > 0) ? new LanguageWorkerChannelException(exMessage, new AggregateException(state.Errors.ToList())) : new LanguageWorkerChannelException(exMessage);
var errorBlock = new ActionBlock <ScriptInvocationContext>(ctx =>
{
ctx.ResultSource.TrySetException(languageWorkerChannelException);
});
_workerStateSubscriptions.Add(state.Functions.Subscribe(reg =>
{
state.AddRegistration(reg);
reg.InputBuffer.LinkTo(errorBlock);
}));
_eventManager.Publish(new WorkerProcessErrorEvent(state.Channel.Id, config.Language, languageWorkerChannelException));
}
return(state);
});
}
}
public void Log <TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func <TState, Exception, string> formatter)
{
if (!IsEnabled(logLevel) || _isUserFunction)
{
return;
}
// enumerate all the state values once, capturing the values we'll use below
// last one wins
string stateSourceValue = null;
string stateFunctionName = null;
string stateEventName = null;
string stateActivityId = null;
if (state is IEnumerable <KeyValuePair <string, object> > stateProps)
{
foreach (var kvp in stateProps)
{
if (string.Equals(kvp.Key, ScriptConstants.LogPropertySourceKey, StringComparison.OrdinalIgnoreCase))
{
stateSourceValue = kvp.Value?.ToString();
}
else if (string.Equals(kvp.Key, ScriptConstants.LogPropertyIsUserLogKey, StringComparison.OrdinalIgnoreCase))
{
if ((bool)kvp.Value)
{
return;
}
}
else if (Utility.IsFunctionName(kvp))
{
stateFunctionName = kvp.Value?.ToString();
}
else if (string.Equals(kvp.Key, ScriptConstants.LogPropertyEventNameKey, StringComparison.OrdinalIgnoreCase))
{
stateEventName = kvp.Value?.ToString();
}
else if (string.Equals(kvp.Key, ScriptConstants.LogPropertyActivityIdKey, StringComparison.OrdinalIgnoreCase))
{
stateActivityId = kvp.Value?.ToString();
}
}
}
// propagate special exceptions through the EventManager
string source = _categoryName ?? stateSourceValue;
if (exception is FunctionIndexingException && _eventManager != null)
{
_eventManager.Publish(new FunctionIndexingEvent(nameof(FunctionIndexingException), source, exception));
}
// If we don't have a message, there's nothing to log.
string formattedMessage = formatter?.Invoke(state, exception);
if (string.IsNullOrEmpty(formattedMessage))
{
return;
}
var scopeProps = _scopeProvider.GetScopeDictionaryOrNull();
string functionName = _functionName ?? stateFunctionName ?? string.Empty;
if (string.IsNullOrEmpty(functionName) && scopeProps?.Count > 0)
{
if (Utility.TryGetFunctionName(scopeProps, out string scopeFunctionName))
{
functionName = scopeFunctionName;
}
}
string invocationId = string.Empty;
object scopeValue = null;
if (scopeProps != null && scopeProps.TryGetValue(ScriptConstants.LogPropertyFunctionInvocationIdKey, out scopeValue) && scopeValue != null)
{
invocationId = scopeValue.ToString();
}
// Apply standard event properties
// Note: we must be sure to default any null values to empty string
// otherwise the ETW event will fail to be persisted (silently)
string summary = formattedMessage ?? string.Empty;
string eventName = !string.IsNullOrEmpty(eventId.Name) ? eventId.Name : stateEventName ?? string.Empty;
string activityId = stateActivityId ?? string.Empty;
string subscriptionId = _appServiceOptions.CurrentValue.SubscriptionId ?? string.Empty;
string appName = _appServiceOptions.CurrentValue.AppName ?? string.Empty;
string runtimeSiteName = _appServiceOptions.CurrentValue.RuntimeSiteName ?? string.Empty;
string slotName = _appServiceOptions.CurrentValue.SlotName ?? string.Empty;
string innerExceptionType = string.Empty;
string innerExceptionMessage = string.Empty;
string details = string.Empty;
if (exception != null)
{
// Populate details from the exception.
if (string.IsNullOrEmpty(functionName) && exception is FunctionInvocationException fex)
{
functionName = string.IsNullOrEmpty(fex.MethodName) ? string.Empty : fex.MethodName.Replace("Host.Functions.", string.Empty);
}
(innerExceptionType, innerExceptionMessage, details) = exception.GetExceptionDetails();
formattedMessage = Sanitizer.Sanitize(formattedMessage);
innerExceptionMessage = innerExceptionMessage ?? string.Empty;
}
_eventGenerator.LogFunctionTraceEvent(logLevel, subscriptionId, appName, functionName, eventName, source, details, summary, innerExceptionType, innerExceptionMessage, invocationId, _hostInstanceId, activityId, runtimeSiteName, slotName, DateTime.UtcNow);
}
private void SendStreamingMessage(StreamingMessage msg)
{
_eventManager.Publish(new OutboundEvent(_workerId, msg));
}
internal void PublishWorkerProcessReadyEvent(FunctionEnvironmentReloadResponse res)
{
WorkerProcessReadyEvent wpEvent = new WorkerProcessReadyEvent(_workerId, _workerConfig.Language);
_eventManager.Publish(wpEvent);
}
public void RaiseWorkerError()
{
Exception testEx = new Exception("Test Worker Error");
_eventManager.Publish(new WorkerErrorEvent(_runtime, Id, testEx));
}
