internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, ILanguageWorkerConsoleLogSource consoleLogSource)
{
_processRegistry = processRegistry;
_workerProcessLogger = workerProcessLogger;
_consoleLogSource = consoleLogSource;
_eventManager = eventManager;
}
internal LanguageWorkerProcess(string runtime,
string workerId,
string rootScriptPath,
Uri serverUri,
WorkerProcessArguments workerProcessArguments,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
ILogger workerProcessLogger,
ILanguageWorkerConsoleLogSource consoleLogSource)
{
_runtime = runtime;
_workerId = workerId;
_processFactory = processFactory;
_processRegistry = processRegistry;
_workerProcessLogger = workerProcessLogger;
_consoleLogSource = consoleLogSource;
_eventManager = eventManager;
var workerContext = new WorkerContext()
{
RequestId = Guid.NewGuid().ToString(),
MaxMessageLength = LanguageWorkerConstants.DefaultMaxMessageLengthBytes,
WorkerId = _workerId,
Arguments = workerProcessArguments,
WorkingDirectory = rootScriptPath,
ServerUri = serverUri,
};
_process = _processFactory.CreateWorkerProcess(workerContext);
}
public LanguageWorkerChannelManager(IScriptEventManager eventManager, IEnvironment environment, IRpcServer rpcServer, ILoggerFactory loggerFactory, IOptions <LanguageWorkerOptions> languageWorkerOptions,
IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, ILanguageWorkerConsoleLogSource consoleLogSource)
{
_rpcServer = rpcServer;
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
_eventManager = eventManager;
_loggerFactory = loggerFactory;
_logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryLanguageWorkerChannelManager);
_workerConfigs = languageWorkerOptions.Value.WorkerConfigs;
_applicationHostOptions = applicationHostOptions;
_consoleLogSource = consoleLogSource;
_processFactory = new DefaultWorkerProcessFactory();
try
{
_processRegistry = ProcessRegistryFactory.Create();
}
catch (Exception e)
{
_logger.LogWarning(e, "Unable to create process registry");
}
_shutdownStandbyWorkerChannels = ScheduleShutdownStandbyChannels;
_shutdownStandbyWorkerChannels = _shutdownStandbyWorkerChannels.Debounce(5000);
_rpcChannelReadySubscriptions = _eventManager.OfType <RpcWebHostChannelReadyEvent>()
.Subscribe(AddOrUpdateWorkerChannels);
}
public async Task WorkerConsoleLogService_ConsoleLogs_LogLevel_Expected(bool useStdErrForErroLogsOnly)
{
_workerConsoleLogSource = new WorkerConsoleLogSource();
_eventManager = new ScriptEventManager();
_processRegistry = new EmptyProcessRegistry();
_workerConsoleLogService = new WorkerConsoleLogService(_testLogger, _workerConsoleLogSource);
WorkerProcess workerProcess = new TestWorkerProcess(_eventManager, _processRegistry, _testLogger, _workerConsoleLogSource, null, useStdErrForErroLogsOnly);
workerProcess.ParseErrorMessageAndLog("Test Message No keyword");
workerProcess.ParseErrorMessageAndLog("Test Error Message");
workerProcess.ParseErrorMessageAndLog("Test Warning Message");
workerProcess.BuildAndLogConsoleLog("LanguageWorkerConsoleLog[Test worker log]", LogLevel.Information);
_ = _workerConsoleLogService.ProcessLogs().ContinueWith(t => { });
await _workerConsoleLogService.StopAsync(System.Threading.CancellationToken.None);
var allLogs = _testLogger.GetLogMessages();
Assert.True(allLogs.Count == 4);
VerifyLogLevel(allLogs, "Test Error Message", LogLevel.Error);
VerifyLogLevel(allLogs, "Test Warning Message", LogLevel.Warning);
if (useStdErrForErroLogsOnly)
{
VerifyLogLevel(allLogs, "Test Message No keyword", LogLevel.Error);
}
else
{
VerifyLogLevel(allLogs, "Test Message No keyword", LogLevel.Information);
}
VerifyLogLevel(allLogs, "[Test worker log]", LogLevel.Debug);
}
internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, bool useStdErrStreamForErrorsOnly = false)
{
_processRegistry = processRegistry;
_workerProcessLogger = workerProcessLogger;
_consoleLogSource = consoleLogSource;
_eventManager = eventManager;
_metricsLogger = metricsLogger;
_useStdErrorStreamForErrorsOnly = useStdErrStreamForErrorsOnly;
}
public HttpWorkerProcessFactory(IScriptEventManager eventManager,
ILoggerFactory loggerFactory,
IWorkerProcessFactory defaultWorkerProcessFactory,
IProcessRegistry processRegistry,
ILanguageWorkerConsoleLogSource consoleLogSource)
{
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_eventManager = eventManager ?? throw new ArgumentNullException(nameof(eventManager));
_consoleLogSource = consoleLogSource ?? throw new ArgumentNullException(nameof(consoleLogSource));
_workerProcessFactory = defaultWorkerProcessFactory ?? throw new ArgumentNullException(nameof(defaultWorkerProcessFactory));
_processRegistry = processRegistry ?? throw new ArgumentNullException(nameof(processRegistry));
}
internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, bool useStdErrStreamForErrorsOnly = false)
{
_processRegistry = processRegistry;
_workerProcessLogger = workerProcessLogger;
_consoleLogSource = consoleLogSource;
_eventManager = eventManager;
_metricsLogger = metricsLogger;
_useStdErrorStreamForErrorsOnly = useStdErrStreamForErrorsOnly;
_serviceProvider = serviceProvider;
// We subscribe to host start events so we can handle the restart that occurs
// on host specialization.
_eventSubscription = _eventManager.OfType <HostStartEvent>().Subscribe(OnHostStart);
}
public LanguageWorkerChannel(
ScriptJobHostOptions scriptConfig,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
IObservable <FunctionRegistrationContext> functionRegistrations,
WorkerConfig workerConfig,
Uri serverUri,
ILoggerFactory loggerFactory,
IMetricsLogger metricsLogger,
int attemptCount)
{
_workerId = Guid.NewGuid().ToString();
_scriptConfig = scriptConfig;
_eventManager = eventManager;
_processFactory = processFactory;
_processRegistry = processRegistry;
_functionRegistrations = functionRegistrations;
_workerConfig = workerConfig;
_serverUri = serverUri;
_workerChannelLogger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}");
_userLogsConsoleLogger = loggerFactory.CreateLogger(LanguageWorkerConstants.FunctionConsoleLogCategoryName);
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.MessageType == MsgType.RpcLog)
.Subscribe(Log));
_eventSubscriptions.Add(_eventManager.OfType <RpcEvent>()
.Where(msg => msg.WorkerId == _workerId)
.Subscribe(msg =>
{
var jsonMsg = JsonConvert.SerializeObject(msg, _verboseSerializerSettings);
_userLogsConsoleLogger.LogTrace(jsonMsg);
}));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => Config.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_startLatencyMetric = metricsLogger.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Language, attemptCount));
StartWorker();
}
public LanguageWorkerChannel(
ScriptHostConfiguration scriptConfig,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
IObservable <FunctionRegistrationContext> functionRegistrations,
WorkerConfig workerConfig,
Uri serverUri,
ILoggerFactory loggerFactory)
{
_workerId = Guid.NewGuid().ToString();
_scriptConfig = scriptConfig;
_eventManager = eventManager;
_processFactory = processFactory;
_processRegistry = processRegistry;
_functionRegistrations = functionRegistrations;
_workerConfig = workerConfig;
_serverUri = serverUri;
_logger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}");
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.MessageType == MsgType.RpcLog)
.Subscribe(Log));
if (scriptConfig.LogFilter.Filter("Worker", LogLevel.Trace))
{
_eventSubscriptions.Add(_eventManager.OfType <RpcEvent>()
.Where(msg => msg.WorkerId == _workerId)
.Subscribe(msg =>
{
var jsonMsg = JsonConvert.SerializeObject(msg, _verboseSerializerSettings);
// TODO: change to trace when ILogger & TraceWriter merge (issues with file trace writer)
_logger.LogInformation(jsonMsg);
}));
}
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => Path.GetExtension(msg.FileChangeArguments.FullPath) == Config.Extension)
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
StartWorker();
}
internal LanguageWorkerChannel(
string workerId,
string rootScriptPath,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
WorkerConfig workerConfig,
Uri serverUri,
ILoggerFactory loggerFactory,
IMetricsLogger metricsLogger,
int attemptCount,
ILanguageWorkerConsoleLogSource consoleLogSource,
bool isWebHostChannel = false,
IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
{
_workerId = workerId;
_rootScriptPath = rootScriptPath;
_eventManager = eventManager;
_processFactory = processFactory;
_processRegistry = processRegistry;
_workerConfig = workerConfig;
_serverUri = serverUri;
_workerChannelLogger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}");
_consoleLogSource = consoleLogSource;
_isWebHostChannel = isWebHostChannel;
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.MessageType == MsgType.RpcLog)
.Subscribe(Log));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => Config.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.FunctionLoadResponse)
.Subscribe((msg) => LoadResponse(msg.Message.FunctionLoadResponse)));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.InvocationResponse)
.Subscribe((msg) => InvokeResponse(msg.Message.InvocationResponse)));
_startLatencyMetric = metricsLogger?.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Language, attemptCount));
_managedDependencyOptions = managedDependencyOptions;
_state = LanguageWorkerChannelState.Default;
}
public RpcWorkerProcessFactory(IRpcServer rpcServer,
IScriptEventManager eventManager,
ILoggerFactory loggerFactory,
IWorkerProcessFactory defaultWorkerProcessFactory,
IProcessRegistry processRegistry,
IWorkerConsoleLogSource consoleLogSource,
IMetricsLogger metricsLogger)
{
_loggerFactory = loggerFactory;
_eventManager = eventManager;
_rpcServer = rpcServer;
_consoleLogSource = consoleLogSource;
_workerProcessFactory = defaultWorkerProcessFactory;
_processRegistry = processRegistry;
_metricsLogger = metricsLogger;
}
public LanguageWorkerProcessFactory(IRpcServer rpcServer,
IOptions <LanguageWorkerOptions> languageWorkerOptions,
IScriptEventManager eventManager,
ILoggerFactory loggerFactory,
IWorkerProcessFactory defaultWorkerProcessFactory,
IProcessRegistry processRegistry,
ILanguageWorkerConsoleLogSource consoleLogSource)
{
_loggerFactory = loggerFactory;
_eventManager = eventManager;
_rpcServer = rpcServer;
_workerConfigs = languageWorkerOptions.Value.WorkerConfigs;
_consoleLogSource = consoleLogSource;
_workerProcessFactory = defaultWorkerProcessFactory;
_processRegistry = processRegistry;
}
public HttpWorkerProcessFactory(IScriptEventManager eventManager,
ILoggerFactory loggerFactory,
IWorkerProcessFactory defaultWorkerProcessFactory,
IProcessRegistry processRegistry,
IWorkerConsoleLogSource consoleLogSource,
IEnvironment environment,
IMetricsLogger metricsLogger)
{
_loggerFactory = loggerFactory ?? throw new ArgumentNullException(nameof(loggerFactory));
_eventManager = eventManager ?? throw new ArgumentNullException(nameof(eventManager));
_consoleLogSource = consoleLogSource ?? throw new ArgumentNullException(nameof(consoleLogSource));
_workerProcessFactory = defaultWorkerProcessFactory ?? throw new ArgumentNullException(nameof(defaultWorkerProcessFactory));
_processRegistry = processRegistry ?? throw new ArgumentNullException(nameof(processRegistry));
_metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
_environment = environment;
}
internal HttpWorkerProcess(string workerId,
string rootScriptPath,
WorkerProcessArguments workerProcessArguments,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
ILogger workerProcessLogger,
ILanguageWorkerConsoleLogSource consoleLogSource)
: base(eventManager, processRegistry, workerProcessLogger, consoleLogSource)
{
_processFactory = processFactory;
_eventManager = eventManager;
_workerProcessLogger = workerProcessLogger;
_workerId = workerId;
_scriptRootPath = rootScriptPath;
_workerProcessArguments = workerProcessArguments;
}
public async Task WorkerConsoleLogService_ConsoleLogs_LogLevel_Expected(bool useStdErrForErrorLogsOnly)
{
// Arrange
_workerConsoleLogSource = new WorkerConsoleLogSource();
_eventManager = new ScriptEventManager();
_processRegistry = new EmptyProcessRegistry();
_workerConsoleLogService = new WorkerConsoleLogService(_testUserLogger, _workerConsoleLogSource);
_serviceProviderMock = new Mock <IServiceProvider>(MockBehavior.Strict);
WorkerProcess workerProcess = new TestWorkerProcess(_eventManager, _processRegistry, _testSystemLogger, _workerConsoleLogSource, null, _serviceProviderMock.Object, useStdErrForErrorLogsOnly);
workerProcess.ParseErrorMessageAndLog("Test Message No keyword");
workerProcess.ParseErrorMessageAndLog("Test Error Message");
workerProcess.ParseErrorMessageAndLog("Test Warning Message");
workerProcess.ParseErrorMessageAndLog("LanguageWorkerConsoleLog[Test Worker Message No keyword]");
workerProcess.ParseErrorMessageAndLog("LanguageWorkerConsoleLog[Test Worker Error Message]");
workerProcess.ParseErrorMessageAndLog("LanguageWorkerConsoleLog[Test Worker Warning Message]");
// Act
_ = _workerConsoleLogService.ProcessLogs().ContinueWith(t => { });
await _workerConsoleLogService.StopAsync(System.Threading.CancellationToken.None);
var userLogs = _testUserLogger.GetLogMessages();
var systemLogs = _testSystemLogger.GetLogMessages();
// Assert
Assert.True(userLogs.Count == 3);
Assert.True(systemLogs.Count == 3);
VerifyLogLevel(userLogs, "Test Error Message", LogLevel.Error);
VerifyLogLevel(systemLogs, "[Test Worker Error Message]", LogLevel.Error);
VerifyLogLevel(userLogs, "Test Warning Message", LogLevel.Warning);
VerifyLogLevel(systemLogs, "[Test Worker Warning Message]", LogLevel.Warning);
if (useStdErrForErrorLogsOnly)
{
VerifyLogLevel(userLogs, "Test Message No keyword", LogLevel.Error);
VerifyLogLevel(systemLogs, "[Test Worker Message No keyword]", LogLevel.Error);
}
else
{
VerifyLogLevel(userLogs, "Test Message No keyword", LogLevel.Information);
VerifyLogLevel(systemLogs, "[Test Worker Message No keyword]", LogLevel.Information);
}
}
private async Task InitializeWorkersAsync()
{
var serverImpl = new FunctionRpcService(EventManager, _logger);
var server = new GrpcServer(serverImpl, ScriptOptions.MaxMessageLengthBytes);
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupGrpcServerLatency))
{
await server.StartAsync();
}
var processFactory = new DefaultWorkerProcessFactory();
try
{
_processRegistry = ProcessRegistryFactory.Create();
}
catch (Exception e)
{
_logger.LogWarning(e, "Unable to create process registry");
}
CreateChannel channelFactory = (languageWorkerConfig, registrations, attemptCount) =>
{
return(new LanguageWorkerChannel(
ScriptOptions,
EventManager,
processFactory,
_processRegistry,
registrations,
languageWorkerConfig,
server.Uri,
_loggerFactory, // TODO: DI (FACAVAL) Pass appropriate logger. Channel facory should likely be a service.
_metricsLogger,
attemptCount));
};
_functionDispatcher = new FunctionDispatcher(EventManager, server, channelFactory, _workerConfigs, _language);
_eventSubscriptions.Add(EventManager.OfType <WorkerProcessErrorEvent>()
.Subscribe(evt =>
{
HandleHostError(evt.Exception);
}));
}
internal HttpWorkerProcess(string workerId,
string rootScriptPath,
HttpWorkerOptions httpWorkerOptions,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
ILogger workerProcessLogger,
IWorkerConsoleLogSource consoleLogSource,
IEnvironment environment,
IMetricsLogger metricsLogger)
: base(eventManager, processRegistry, workerProcessLogger, consoleLogSource, metricsLogger, httpWorkerOptions.Description.UseStdErrorStreamForErrorsOnly)
{
_processFactory = processFactory;
_eventManager = eventManager;
_workerProcessLogger = workerProcessLogger;
_workerId = workerId;
_scriptRootPath = rootScriptPath;
_httpWorkerOptions = httpWorkerOptions;
_workerProcessArguments = _httpWorkerOptions.Arguments;
_environment = environment;
}
internal RpcWorkerProcess(string runtime,
string workerId,
string rootScriptPath,
Uri serverUri,
WorkerProcessArguments workerProcessArguments,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
ILogger workerProcessLogger,
IWorkerConsoleLogSource consoleLogSource)
: base(eventManager, processRegistry, workerProcessLogger, consoleLogSource)
{
_runtime = runtime;
_processFactory = processFactory;
_eventManager = eventManager;
_workerProcessLogger = workerProcessLogger;
_workerId = workerId;
_serverUri = serverUri;
_scriptRootPath = rootScriptPath;
_workerProcessArguments = workerProcessArguments;
}
internal RpcWorkerProcess(string runtime,
string workerId,
string rootScriptPath,
Uri serverUri,
RpcWorkerConfig rpcWorkerConfig,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
ILogger workerProcessLogger,
IWorkerConsoleLogSource consoleLogSource,
IMetricsLogger metricsLogger)
: base(eventManager, processRegistry, workerProcessLogger, consoleLogSource, metricsLogger, rpcWorkerConfig.Description.UseStdErrorStreamForErrorsOnly)
{
_runtime = runtime;
_processFactory = processFactory;
_eventManager = eventManager;
_workerProcessLogger = workerProcessLogger;
_workerId = workerId;
_serverUri = serverUri;
_scriptRootPath = rootScriptPath;
_workerProcessArguments = rpcWorkerConfig.Arguments;
_workerDirectory = rpcWorkerConfig.Description.WorkerDirectory;
}
internal TestWorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, bool useStdErrStreamForErrorsOnly = false)
: base(eventManager, processRegistry, workerProcessLogger, consoleLogSource, metricsLogger, serviceProvider, useStdErrStreamForErrorsOnly)
{
}
public ClientHostProcessRegistry(IProcessRegistry originalInstance)
{
_originalInstance = originalInstance;
}