public void WorkerIndexingDecisionLogic_NullWorkerIndexingProperty(bool workerIndexingFeatureFlag, bool expected)
{
var testEnv = new TestEnvironment();
testEnv.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, RpcWorkerConstants.PythonLanguageWorkerName);
if (workerIndexingFeatureFlag)
{
testEnv.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing);
}
RpcWorkerConfig workerConfig = new RpcWorkerConfig()
{
Description = new RpcWorkerDescription()
{
Extensions = new List <string>(),
Language = "python",
WorkerDirectory = "testDir",
WorkerIndexing = null
}
};
bool workerShouldIndex = Utility.CanWorkerIndex(new List <RpcWorkerConfig>()
{
workerConfig
}, testEnv);
Assert.Equal(expected, workerShouldIndex);
}
public void ReadWorkerProviderFromConfig_Concatenate_ArgsFromSettings_ArgsFromWorkerConfig()
{
string[] argsFromConfig = new string[] { "--expose-http2", "--no-deprecation" };
var configs = new List <TestRpcWorkerConfig>()
{
MakeTestConfig(testLanguage, argsFromConfig)
};
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
// Creates temp directory w/ worker.config.json and runs ReadWorkerProviderFromConfig
Dictionary <string, string> keyValuePairs = new Dictionary <string, string>
{
[$"{RpcWorkerConstants.LanguageWorkersSectionName}:{testLanguage}:{WorkerConstants.WorkerDescriptionArguments}"] = "--inspect=5689"
};
var workerConfigs = TestReadWorkerProviderFromConfig(configs, new TestLogger(testLanguage), testMetricsLogger, null, keyValuePairs);
AreRequiredMetricsEmitted(testMetricsLogger);
Assert.Single(workerConfigs);
RpcWorkerConfig workerConfig = workerConfigs.Single();
Assert.Equal(Path.Combine(rootPath, testLanguage, $"{RpcWorkerConfigTestUtilities.TestWorkerPathInWorkerConfig}.{testLanguage}"), workerConfig.Description.DefaultWorkerPath);
Assert.True(workerConfig.Description.Arguments.Count == 3);
Assert.True(workerConfig.Description.Arguments.Contains("--inspect=5689"));
Assert.True(workerConfig.Description.Arguments.Contains("--no-deprecation"));
Assert.True(workerConfig.Description.Arguments.Contains("--expose-http2"));
}
public RpcWorkerChannelTests()
{
_logger = new TestLogger("FunctionDispatcherTests");
_testFunctionRpcService = new TestFunctionRpcService(_eventManager, _workerId, _logger, _expectedLogMsg);
_testWorkerConfig = TestHelpers.GetTestWorkerConfigs().FirstOrDefault();
_mockrpcWorkerProcess.Setup(m => m.StartProcessAsync()).Returns(Task.CompletedTask);
_testEnvironment = new TestEnvironment();
var hostOptions = new ScriptApplicationHostOptions
{
IsSelfHost = true,
ScriptPath = _scriptRootPath,
LogPath = Environment.CurrentDirectory, // not tested
SecretsPath = Environment.CurrentDirectory, // not tested
HasParentScope = true
};
_hostOptionsMonitor = TestHelpers.CreateOptionsMonitor(hostOptions);
_workerChannel = new RpcWorkerChannel(
_workerId,
_eventManager,
_testWorkerConfig,
_mockrpcWorkerProcess.Object,
_logger,
_metricsLogger,
0,
_testEnvironment,
_hostOptionsMonitor);
}
public async Task GetLatencies_DoesNot_StartTimer_WhenDynamicConcurrencyDisabled()
{
RpcWorkerConfig config = new RpcWorkerConfig()
{
Description = new RpcWorkerDescription()
{
Language = RpcWorkerConstants.NodeLanguageWorkerName
},
};
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerDynamicConcurrencyEnabled, null);
GrpcWorkerChannel workerChannel = new GrpcWorkerChannel(
_workerId,
_eventManager,
config,
_mockrpcWorkerProcess.Object,
_logger,
_metricsLogger,
0,
_testEnvironment,
_hostOptionsMonitor,
_sharedMemoryManager,
_functionDataCache,
_workerConcurrencyOptions);
// wait 10 seconds
await Task.Delay(10000);
IEnumerable <TimeSpan> latencyHistory = workerChannel.GetLatencies();
Assert.True(latencyHistory.Count() == 0);
}
internal GrpcWorkerChannel(
string workerId,
IScriptEventManager eventManager,
RpcWorkerConfig workerConfig,
IWorkerProcess rpcWorkerProcess,
ILogger logger,
IMetricsLogger metricsLogger,
int attemptCount,
IEnvironment environment,
IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions,
ISharedMemoryManager sharedMemoryManager,
IFunctionDataCache functionDataCache,
IOptions <WorkerConcurrencyOptions> workerConcurrencyOptions)
{
_workerId = workerId;
_eventManager = eventManager;
_workerConfig = workerConfig;
_runtime = workerConfig.Description.Language;
_rpcWorkerProcess = rpcWorkerProcess;
_workerChannelLogger = logger;
_metricsLogger = metricsLogger;
_environment = environment;
_applicationHostOptions = applicationHostOptions;
_sharedMemoryManager = sharedMemoryManager;
_workerConcurrencyOptions = workerConcurrencyOptions;
_workerCapabilities = new GrpcCapabilities(_workerChannelLogger);
_inboundWorkerEvents = _eventManager.OfType <InboundGrpcEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.IsMessageOfType(MsgType.RpcLog) && !msg.IsLogOfCategory(RpcLogCategory.System))
.Subscribe(Log));
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.IsMessageOfType(MsgType.RpcLog) && msg.IsLogOfCategory(RpcLogCategory.System))
.Subscribe(SystemLog));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => _workerConfig.Description.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_eventSubscriptions.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.InvocationResponse)
.Subscribe(async(msg) => await InvokeResponse(msg.Message.InvocationResponse)));
_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.WorkerStatusResponse)
.Subscribe((msg) => ReceiveWorkerStatusResponse(msg.Message.RequestId, msg.Message.WorkerStatusResponse));
_startLatencyMetric = metricsLogger?.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Description.Language, attemptCount));
_state = RpcWorkerChannelState.Default;
}
public GrpcWorkerChannelTests()
{
_logger = new TestLogger("FunctionDispatcherTests");
_testFunctionRpcService = new TestFunctionRpcService(_eventManager, _workerId, _logger, _expectedLogMsg);
_testWorkerConfig = TestHelpers.GetTestWorkerConfigs().FirstOrDefault();
_testWorkerConfig.CountOptions.ProcessStartupTimeout = TimeSpan.FromSeconds(5);
_testWorkerConfig.CountOptions.InitializationTimeout = TimeSpan.FromSeconds(5);
_testWorkerConfig.CountOptions.EnvironmentReloadTimeout = TimeSpan.FromSeconds(5);
_mockrpcWorkerProcess.Setup(m => m.StartProcessAsync()).Returns(Task.CompletedTask);
_mockrpcWorkerProcess.Setup(m => m.Id).Returns(910);
_testEnvironment = new TestEnvironment();
_testEnvironment.SetEnvironmentVariable(FunctionDataCacheConstants.FunctionDataCacheEnabledSettingName, "1");
_workerConcurrencyOptions = Options.Create(new WorkerConcurrencyOptions());
_workerConcurrencyOptions.Value.CheckInterval = TimeSpan.FromSeconds(1);
ILogger <MemoryMappedFileAccessor> mmapAccessorLogger = NullLogger <MemoryMappedFileAccessor> .Instance;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_mapAccessor = new MemoryMappedFileAccessorWindows(mmapAccessorLogger);
}
else
{
_mapAccessor = new MemoryMappedFileAccessorUnix(mmapAccessorLogger, _testEnvironment);
}
_sharedMemoryManager = new SharedMemoryManager(_loggerFactory, _mapAccessor);
_functionDataCache = new FunctionDataCache(_sharedMemoryManager, _loggerFactory, _testEnvironment);
var hostOptions = new ScriptApplicationHostOptions
{
IsSelfHost = true,
ScriptPath = _scriptRootPath,
LogPath = Environment.CurrentDirectory, // not tested
SecretsPath = Environment.CurrentDirectory, // not tested
HasParentScope = true
};
_hostOptionsMonitor = TestHelpers.CreateOptionsMonitor(hostOptions);
_workerChannel = new GrpcWorkerChannel(
_workerId,
_eventManager,
_testWorkerConfig,
_mockrpcWorkerProcess.Object,
_logger,
_metricsLogger,
0,
_testEnvironment,
_hostOptionsMonitor,
_sharedMemoryManager,
_functionDataCache,
_workerConcurrencyOptions);
}
public void ReadWorkerProviderFromConfig_ReturnsProviderNoArguments()
{
var configs = new List <TestRpcWorkerConfig>()
{
MakeTestConfig(testLanguage, new string[0])
};
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
// Creates temp directory w/ worker.config.json and runs ReadWorkerProviderFromConfig
var workerConfigs = TestReadWorkerProviderFromConfig(configs, new TestLogger(testLanguage), testMetricsLogger);
AreRequiredMetricsEmitted(testMetricsLogger);
Assert.Single(workerConfigs);
Assert.Equal(Path.Combine(rootPath, testLanguage, $"{RpcWorkerConfigTestUtilities.TestWorkerPathInWorkerConfig}.{testLanguage}"), workerConfigs.Single().Description.DefaultWorkerPath);
RpcWorkerConfig worker = workerConfigs.FirstOrDefault();
Assert.True(worker.Description.Arguments.Count == 0);
}
public RpcWorkerChannelTests()
{
_logger = new TestLogger("FunctionDispatcherTests");
_testFunctionRpcService = new TestFunctionRpcService(_eventManager, _workerId, _logger, _expectedLogMsg);
_testWorkerConfig = TestHelpers.GetTestWorkerConfigs().FirstOrDefault();
_mockrpcWorkerProcess.Setup(m => m.StartProcessAsync()).Returns(Task.CompletedTask);
_workerChannel = new RpcWorkerChannel(
_workerId,
_scriptRootPath,
_eventManager,
_testWorkerConfig,
_mockrpcWorkerProcess.Object,
_logger,
_metricsLogger,
0);
}
public GrpcWorkerChannelTests()
{
_logger = new TestLogger("FunctionDispatcherTests");
_testFunctionRpcService = new TestFunctionRpcService(_eventManager, _workerId, _logger, _expectedLogMsg);
_testWorkerConfig = TestHelpers.GetTestWorkerConfigs().FirstOrDefault();
_mockrpcWorkerProcess.Setup(m => m.StartProcessAsync()).Returns(Task.CompletedTask);
_testEnvironment = new TestEnvironment();
ILogger <MemoryMappedFileAccessor> mmapAccessorLogger = NullLogger <MemoryMappedFileAccessor> .Instance;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
_mapAccessor = new MemoryMappedFileAccessorWindows(mmapAccessorLogger);
}
else
{
_mapAccessor = new MemoryMappedFileAccessorUnix(mmapAccessorLogger, _testEnvironment);
}
_sharedMemoryManager = new SharedMemoryManager(_loggerFactory, _mapAccessor);
var hostOptions = new ScriptApplicationHostOptions
{
IsSelfHost = true,
ScriptPath = _scriptRootPath,
LogPath = Environment.CurrentDirectory, // not tested
SecretsPath = Environment.CurrentDirectory, // not tested
HasParentScope = true
};
_hostOptionsMonitor = TestHelpers.CreateOptionsMonitor(hostOptions);
_workerChannel = new GrpcWorkerChannel(
_workerId,
_eventManager,
_testWorkerConfig,
_mockrpcWorkerProcess.Object,
_logger,
_metricsLogger,
0,
_testEnvironment,
_hostOptionsMonitor,
_sharedMemoryManager);
}
public void ReadWorkerProviderFromConfig_ReturnsProviderWithArguments()
{
var expectedArguments = new string[] { "-v", "verbose" };
var configs = new List <TestRpcWorkerConfig>()
{
MakeTestConfig(testLanguage, expectedArguments)
};
var testLogger = new TestLogger(testLanguage);
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
// Creates temp directory w/ worker.config.json and runs ReadWorkerProviderFromConfig
IEnumerable <RpcWorkerConfig> workerConfigs = TestReadWorkerProviderFromConfig(configs, testLogger, testMetricsLogger);
AreRequiredMetricsEmitted(testMetricsLogger);
Assert.Single(workerConfigs);
RpcWorkerConfig worker = workerConfigs.FirstOrDefault();
Assert.True(expectedArguments.SequenceEqual(worker.Description.Arguments.ToArray()));
}
public void ReadWorkerProviderFromConfig_AddProfile_ReturnsDefaultDescription()
{
var expectedArguments = new string[] { "-v", "verbose" };
var configs = new List <TestRpcWorkerConfig>()
{
MakeTestConfig(testLanguage, expectedArguments, false, "TestProfile")
};
var testLogger = new TestLogger(testLanguage);
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
// Creates temp directory w/ worker.config.json and runs ReadWorkerProviderFromConfig
IEnumerable <RpcWorkerConfig> workerConfigs = TestReadWorkerProviderFromConfig(configs, testLogger, testMetricsLogger);
AreRequiredMetricsEmitted(testMetricsLogger);
Assert.Single(workerConfigs);
RpcWorkerConfig worker = workerConfigs.FirstOrDefault();
Assert.Equal(RpcWorkerConfigTestUtilities.TestDefaultExecutablePath, worker.Description.DefaultExecutablePath);
}
public void VerifyWorkerIndexingDecisionLogic(bool workerIndexingFeatureFlag, bool workerIndexingConfigProperty, bool expected)
{
var testEnv = new TestEnvironment();
testEnv.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, RpcWorkerConstants.PythonLanguageWorkerName);
if (workerIndexingFeatureFlag)
{
testEnv.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing);
}
RpcWorkerConfig workerConfig = new RpcWorkerConfig()
{
Description = TestHelpers.GetTestWorkerDescription("python", "none", workerIndexingConfigProperty)
};
bool workerShouldIndex = Utility.CanWorkerIndex(new List <RpcWorkerConfig>()
{
workerConfig
}, testEnv);
Assert.Equal(expected, workerShouldIndex);
}
public void ReadWorkerProviderFromConfig_OverrideDefaultExePath()
{
var configs = new List <TestRpcWorkerConfig>()
{
MakeTestConfig(testLanguage, new string[0], false, WorkerConstants.WorkerDescriptionAppServiceEnvProfileName)
};
var testLogger = new TestLogger(testLanguage);
var testExePath = "./mySrc/myIndex";
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
Dictionary <string, string> keyValuePairs = new Dictionary <string, string>
{
[$"{RpcWorkerConstants.LanguageWorkersSectionName}:{testLanguage}:{WorkerConstants.WorkerDescriptionDefaultExecutablePath}"] = testExePath
};
var workerConfigs = TestReadWorkerProviderFromConfig(configs, new TestLogger(testLanguage), testMetricsLogger, null, keyValuePairs, true);
AreRequiredMetricsEmitted(testMetricsLogger);
Assert.Single(workerConfigs);
RpcWorkerConfig worker = workerConfigs.FirstOrDefault();
Assert.Equal(testExePath, worker.Description.DefaultExecutablePath);
}
public void ReadWorkerProviderFromAppSetting()
{
var testConfig = MakeTestConfig(testLanguage, new string[0]);
var configs = new List <TestRpcWorkerConfig>()
{
testConfig
};
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
RpcWorkerConfigTestUtilities.CreateWorkerFolder(customRootPath, testConfig);
Dictionary <string, string> keyValuePairs = new Dictionary <string, string>
{
[$"{RpcWorkerConstants.LanguageWorkersSectionName}:{testLanguage}:{WorkerConstants.WorkerDirectorySectionName}"] = Path.Combine(customRootPath, testLanguage)
};
var workerConfigs = TestReadWorkerProviderFromConfig(configs, new TestLogger(testLanguage), testMetricsLogger, null, keyValuePairs);
AreRequiredMetricsEmitted(testMetricsLogger);
Assert.Single(workerConfigs);
RpcWorkerConfig workerConfig = workerConfigs.Single();
Assert.Equal(Path.Combine(customRootPath, testLanguage, $"{RpcWorkerConfigTestUtilities.TestWorkerPathInWorkerConfig}.{testLanguage}"), workerConfig.Description.DefaultWorkerPath);
}
public async Task GetLatencies_StartsTimer_WhenDynamicConcurrencyEnabled()
{
RpcWorkerConfig config = new RpcWorkerConfig()
{
Description = new RpcWorkerDescription()
{
Language = RpcWorkerConstants.NodeLanguageWorkerName
}
};
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerDynamicConcurrencyEnabled, "true");
GrpcWorkerChannel workerChannel = new GrpcWorkerChannel(
_workerId,
_eventManager,
config,
_mockrpcWorkerProcess.Object,
_logger,
_metricsLogger,
0,
_testEnvironment,
_hostOptionsMonitor,
_sharedMemoryManager,
_functionDataCache,
_workerConcurrencyOptions);
IEnumerable <TimeSpan> latencyHistory = null;
// wait 10 seconds
await TestHelpers.Await(() =>
{
latencyHistory = workerChannel.GetLatencies();
return(latencyHistory.Count() > 0);
}, pollingInterval : 1000, timeout : 10 * 1000);
// We have non empty latencyHistory so the timer was started
_testEnvironment.SetEnvironmentVariable(RpcWorkerConstants.FunctionsWorkerDynamicConcurrencyEnabled, null);
}
public async Task GetExtensionsStartupTypes_WorkerIndexing_BypassesSelectiveLoading()
{
var storageExtensionReference = new ExtensionReference {
Name = "Storage", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName
};
storageExtensionReference.Bindings.Add("blob");
var sendGridExtensionReference = new ExtensionReference {
Name = "SendGrid", TypeName = typeof(AzureStorageWebJobsStartup).AssemblyQualifiedName
};
sendGridExtensionReference.Bindings.Add("sendGrid");
var references = new[] { storageExtensionReference, sendGridExtensionReference };
TestMetricsLogger testMetricsLogger = new TestMetricsLogger();
var extensions = new JObject
{
{ "extensions", JArray.FromObject(references) }
};
using (var directory = new TempDirectory())
{
var binPath = Path.Combine(directory.Path, "bin");
Directory.CreateDirectory(binPath);
void CopyToBin(string path)
{
File.Copy(path, Path.Combine(binPath, Path.GetFileName(path)));
}
CopyToBin(typeof(AzureStorageWebJobsStartup).Assembly.Location);
File.WriteAllText(Path.Combine(binPath, "extensions.json"), extensions.ToString());
TestLoggerProvider testLoggerProvider = new TestLoggerProvider();
LoggerFactory factory = new LoggerFactory();
factory.AddProvider(testLoggerProvider);
var testLogger = factory.CreateLogger <ScriptStartupTypeLocator>();
var mockFunctionMetadataManager = GetTestFunctionMetadataManager();
var mockExtensionBundleManager = new Mock <IExtensionBundleManager>();
mockExtensionBundleManager.Setup(e => e.IsExtensionBundleConfigured()).Returns(true);
mockExtensionBundleManager.Setup(e => e.GetExtensionBundleBinPathAsync()).Returns(Task.FromResult(binPath));
// mock worker config and environment variables to make host choose worker indexing
RpcWorkerConfig workerConfig = new RpcWorkerConfig()
{
Description = TestHelpers.GetTestWorkerDescription("python", "none", true)
};
var tempOptions = new LanguageWorkerOptions();
tempOptions.WorkerConfigs = new List <RpcWorkerConfig>();
tempOptions.WorkerConfigs.Add(workerConfig);
var languageWorkerOptions = new OptionsWrapper <LanguageWorkerOptions>(tempOptions);
Environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, ScriptConstants.FeatureFlagEnableWorkerIndexing);
Environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, "python");
var discoverer = new ScriptStartupTypeLocator(directory.Path, testLogger, mockExtensionBundleManager.Object, mockFunctionMetadataManager, testMetricsLogger, languageWorkerOptions);
// Act
var types = await discoverer.GetExtensionsStartupTypesAsync();
Environment.SetEnvironmentVariable(EnvironmentSettingNames.AzureWebJobsFeatureFlags, null);
Environment.SetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime, null);
//Assert that filtering did not take place because of worker indexing
Assert.True(types.Count() == 2);
Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.ElementAt(0).FullName);
Assert.Equal(typeof(AzureStorageWebJobsStartup).FullName, types.ElementAt(1).FullName);
}
}