internal CSharpFunctionInvoker(ScriptHost host, FunctionMetadata functionMetadata,
Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings,
IFunctionEntryPointResolver functionEntryPointResolver, FunctionAssemblyLoader assemblyLoader)
: base(host, functionMetadata)
{
_host = host;
_functionEntryPointResolver = functionEntryPointResolver;
_assemblyLoader = assemblyLoader;
_metadataResolver = new FunctionMetadataResolver(functionMetadata, TraceWriter);
_inputBindings = inputBindings;
_outputBindings = outputBindings;
_triggerInputName = GetTriggerInputName(functionMetadata);
_metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();
InitializeFileWatcherIfEnabled();
_resultProcessor = CreateResultProcessor();
_functionValueLoader = FunctionValueLoader.Create(CreateFunctionTarget);
_reloadScript = ReloadScript;
_reloadScript = _reloadScript.Debounce();
_restorePackages = RestorePackages;
_restorePackages = _restorePackages.Debounce();
}
internal DotNetFunctionInvoker(ScriptHost host, FunctionMetadata functionMetadata,
Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings,
IFunctionEntryPointResolver functionEntryPointResolver, FunctionAssemblyLoader assemblyLoader,
ICompilationServiceFactory compilationServiceFactory, ITraceWriterFactory traceWriterFactory = null)
: base(host, functionMetadata, traceWriterFactory)
{
_metricsLogger = Host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();
_functionEntryPointResolver = functionEntryPointResolver;
_assemblyLoader = assemblyLoader;
_metadataResolver = new FunctionMetadataResolver(functionMetadata, host.ScriptConfig.BindingProviders, TraceWriter);
_compilationService = compilationServiceFactory.CreateService(functionMetadata.ScriptType, _metadataResolver);
_inputBindings = inputBindings;
_outputBindings = outputBindings;
_triggerInputName = functionMetadata.Bindings.FirstOrDefault(b => b.IsTrigger).Name;
InitializeFileWatcher();
_resultProcessor = CreateResultProcessor();
_functionLoader = new FunctionLoader<MethodInfo>(CreateFunctionTarget);
_reloadScript = ReloadScript;
_reloadScript = _reloadScript.Debounce();
_restorePackages = RestorePackages;
_restorePackages = _restorePackages.Debounce();
}
internal ScriptFunctionInvoker(string scriptFilePath, ScriptHost host, FunctionMetadata functionMetadata, Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings)
: base(host, functionMetadata)
{
_scriptFilePath = scriptFilePath;
_inputBindings = inputBindings;
_outputBindings = outputBindings;
_metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();
}
/// <summary>
/// Creates an instance of MetricsInvoker
/// </summary>
/// <param name="filesToProcess">Which files to process, Can be wildcards or explicit file names</param>
/// <param name="rootPath">Root path where to look for binaries</param>
/// <param name="output">The resulting output from metrics.exe</param>
/// <param name="logger">Instance of IMetricsLogger</param>
/// <returns>A MetricsInvoker instance </returns>
public static MetricsInvoker Create(IEnumerable<string> filesToProcess, string rootPath, string output, IMetricsLogger logger)
{
string metricsExePath = Path.Combine(CodeQuality.CodeMetrics.ProgramFilesX86(), @"Microsoft Visual Studio 10.0\Team Tools\Static Analysis Tools\FxCop\metrics.exe");
if (!File.Exists(metricsExePath))
{
logger.LogError("Could not locate " + metricsExePath + ". Please download Visual Studio Code Metrics PowerTool 10.0 at http://www.microsoft.com/downloads/en/details.aspx?FamilyID=edd1dfb0-b9fe-4e90-b6a6-5ed6f6f6e615");
return null;
}
return new MetricsInvoker(metricsExePath, filesToProcess, rootPath, output, logger);
}
internal NodeFunctionInvoker(ScriptHost host, BindingMetadata trigger, FunctionMetadata functionMetadata, Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings)
: base(host, functionMetadata)
{
_trigger = trigger;
string scriptFilePath = functionMetadata.Source.Replace('\\', '/');
_script = string.Format(CultureInfo.InvariantCulture, _functionTemplate, scriptFilePath);
_inputBindings = inputBindings;
_outputBindings = outputBindings;
_metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();
InitializeFileWatcherIfEnabled();
}
private MetricsInvoker(string metricsExePath, IEnumerable<string> filesToProcess, string rootPath, string output, IMetricsLogger logger)
{
this.metricsExePath = metricsExePath;
this.output = output;
this.logger = logger;
this.Argument = string.Empty;
foreach (var file in filesToProcess)
{
this.Argument += string.Format(" /f:\"{0}\"", Path.Combine(rootPath, file));
}
this.Argument += string.Format(" /out:\"{0}\"", output);
this.Argument = this.Argument.Trim();
}
internal FunctionInvokerBase(ScriptHost host, FunctionMetadata functionMetadata, ITraceWriterFactory traceWriterFactory = null)
{
Host = host;
Metadata = functionMetadata;
_metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();
// Function file logging is only done conditionally
traceWriterFactory = traceWriterFactory ?? new FunctionTraceWriterFactory(functionMetadata.Name, Host.ScriptConfig);
TraceWriter traceWriter = traceWriterFactory.Create();
FileTraceWriter = traceWriter.Conditional(t => Host.FileLoggingEnabled && (!(t.Properties?.ContainsKey(ScriptConstants.TracePropertyPrimaryHostKey) ?? false) || Host.IsPrimary));
// The global trace writer used by the invoker will write all traces to both
// the host trace writer as well as our file trace writer
TraceWriter = host.TraceWriter != null ?
new CompositeTraceWriter(new TraceWriter[] { FileTraceWriter, host.TraceWriter }) :
FileTraceWriter;
// Apply the function name as an event property to all traces
var functionTraceProperties = new Dictionary<string, object>
{
{ ScriptConstants.TracePropertyFunctionNameKey, Metadata.Name }
};
TraceWriter = TraceWriter.Apply(functionTraceProperties);
}
public WebScriptHostManager(ScriptHostConfiguration config, SecretManager secretManager) : base(config)
{
_metricsLogger = new WebHostMetricsLogger();
_secretManager = secretManager;
}
public FunctionMetadataProvider(IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, ILogger <FunctionMetadataProvider> logger, IMetricsLogger metricsLogger)
{
_applicationHostOptions = applicationHostOptions;
_metricsLogger = metricsLogger;
_logger = logger;
}
public SecretManager(ISecretsRepository repository, IKeyValueConverterFactory keyValueConverterFactory, ILogger logger, IMetricsLogger metricsLogger, bool createHostSecretsIfMissing = false)
{
_repository = repository;
_keyValueConverterFactory = keyValueConverterFactory;
_repository.SecretsChanged += OnSecretsChanged;
_logger = logger;
_metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
_repositoryClassName = _repository.GetType().Name.ToLower();
if (createHostSecretsIfMissing)
{
// The SecretManager implementation of GetHostSecrets will
// create a host secret if one is not present.
GetHostSecretsAsync().GetAwaiter().GetResult();
}
}
public InstanceManager(IOptionsFactory <ScriptApplicationHostOptions> optionsFactory, HttpClient client, IScriptWebHostEnvironment webHostEnvironment,
IEnvironment environment, ILogger <InstanceManager> logger, IMetricsLogger metricsLogger)
{
_client = client ?? throw new ArgumentNullException(nameof(client));
_webHostEnvironment = webHostEnvironment ?? throw new ArgumentNullException(nameof(webHostEnvironment));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_metricsLogger = metricsLogger;
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
_optionsFactory = optionsFactory ?? throw new ArgumentNullException(nameof(optionsFactory));
}
public ILanguageWorkerChannel Create(string scriptRootPath, string language, IMetricsLogger metricsLogger, int attemptCount, IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
{
return(new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), language, _eventManager, _testLogger, throwOnProcessStartUp: _throwOnProcessStartUp));
}
public DisposableEvent(string eventName, string functionName, IMetricsLogger metricsLogger)
{
_metricEvent = metricsLogger.BeginEvent(eventName, functionName);
_metricsLogger = metricsLogger;
}
public WebScriptHostManager(ScriptHostConfiguration config) : base(config)
{
_metricsLogger = new WebHostMetricsLogger();
}
public DisposableEvent(string eventName, IMetricsLogger metricsLogger)
{
_metricEvent = metricsLogger.BeginEvent(eventName);
_metricsLogger = metricsLogger;
}
public static IHostBuilder AddWebScriptHost(this IHostBuilder builder, IServiceProvider rootServiceProvider,
IServiceScopeFactory rootScopeFactory, ScriptApplicationHostOptions webHostOptions, Action <IWebJobsBuilder> configureWebJobs = null)
{
ILoggerFactory configLoggerFactory = rootServiceProvider.GetService <ILoggerFactory>();
IDependencyValidator validator = rootServiceProvider.GetService <IDependencyValidator>();
IMetricsLogger metricsLogger = rootServiceProvider.GetService <IMetricsLogger>();
builder.UseServiceProviderFactory(new JobHostScopedServiceProviderFactory(rootServiceProvider, rootScopeFactory, validator))
.ConfigureServices(services =>
{
// register default configuration
// must happen before the script host is added below
services.ConfigureOptions <HttpOptionsSetup>();
services.ConfigureOptions <CustomHttpHeadersOptionsSetup>();
services.ConfigureOptions <HostHstsOptionsSetup>();
services.ConfigureOptions <HostCorsOptionsSetup>();
services.ConfigureOptions <CorsOptionsSetup>();
services.ConfigureOptions <AppServiceOptionsSetup>();
services.ConfigureOptions <HostEasyAuthOptionsSetup>();
})
.AddScriptHost(webHostOptions, configLoggerFactory, metricsLogger, webJobsBuilder =>
{
webJobsBuilder.AddAzureStorageCoreServices();
configureWebJobs?.Invoke(webJobsBuilder);
ConfigureRegisteredBuilders(webJobsBuilder, rootServiceProvider);
webJobsBuilder.Services.AddSingleton <IHttpRoutesManager, WebScriptHostHttpRoutesManager>();
})
.ConfigureAppConfiguration(configurationBuilder =>
{
ConfigureRegisteredBuilders(configurationBuilder, rootServiceProvider);
})
.ConfigureLogging(loggingBuilder =>
{
loggingBuilder.Services.AddSingleton <ILoggerFactory, ScriptLoggerFactory>();
loggingBuilder.AddWebJobsSystem <SystemLoggerProvider>();
loggingBuilder.Services.AddSingleton <ILoggerProvider, AzureMonitorDiagnosticLoggerProvider>();
ConfigureRegisteredBuilders(loggingBuilder, rootServiceProvider);
})
.ConfigureServices(services =>
{
var webHostEnvironment = rootServiceProvider.GetService <IScriptWebHostEnvironment>();
var environment = rootServiceProvider.GetService <IEnvironment>();
if (FunctionsSyncManager.IsSyncTriggersEnvironment(webHostEnvironment, environment))
{
services.AddSingleton <IHostedService, FunctionsSyncService>();
}
if (!environment.IsV2CompatibilityMode())
{
new FunctionsMvcBuilder(services).AddNewtonsoftJson();
}
services.AddSingleton <HttpRequestQueue>();
services.AddSingleton <IHostLifetime, JobHostHostLifetime>();
services.AddSingleton <IWebJobsExceptionHandler, WebScriptHostExceptionHandler>();
services.AddSingleton <DefaultScriptWebHookProvider>();
services.TryAddSingleton <IScriptWebHookProvider>(p => p.GetService <DefaultScriptWebHookProvider>());
services.TryAddSingleton <IWebHookProvider>(p => p.GetService <DefaultScriptWebHookProvider>());
services.TryAddSingleton <IJobHostMiddlewarePipeline, DefaultMiddlewarePipeline>();
if (environment.IsLinuxConsumption())
{
services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, JobHostEasyAuthMiddleware>());
}
services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, CustomHttpHeadersMiddleware>());
services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, HstsConfigurationMiddleware>());
if (environment.IsLinuxConsumption())
{
services.AddSingleton <ICorsMiddlewareFactory, CorsMiddlewareFactory>();
services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, JobHostCorsMiddleware>());
}
services.TryAddSingleton <IScaleMetricsRepository, TableStorageScaleMetricsRepository>();
if (environment.IsWindowsAzureManagedHosting() || environment.IsLinuxAzureManagedHosting())
{
// Enable breaking change analysis only when hosted in Azure
services.AddSingleton <IChangeAnalysisStateProvider, BlobChangeAnalysisStateProvider>();
services.AddSingleton <IHostedService, ChangeAnalysisService>();
}
// Make sure the registered IHostIdProvider is used
IHostIdProvider provider = rootServiceProvider.GetService <IHostIdProvider>();
if (provider != null)
{
services.AddSingleton <IHostIdProvider>(provider);
}
services.AddSingleton <IDelegatingHandlerProvider, DefaultDelegatingHandlerProvider>();
// Logging and diagnostics
services.AddSingleton <IMetricsLogger>(a => new NonDisposableMetricsLogger(metricsLogger));
services.AddSingleton <IEventCollectorProvider, FunctionInstanceLogCollectorProvider>();
// Hosted services
services.TryAddEnumerable(ServiceDescriptor.Singleton <IHostedService, FileMonitoringService>());
ConfigureRegisteredBuilders(services, rootServiceProvider);
});
var debugStateProvider = rootServiceProvider.GetService <IDebugStateProvider>();
if (debugStateProvider.InDebugMode)
{
builder.UseEnvironment(EnvironmentName.Development);
}
return(builder);
}
public WebScriptHostRequestManager(HttpConfiguration config, HostPerformanceManager performanceManager, IMetricsLogger metricsLogger, TraceWriter traceWriter, int performanceCheckPeriodSeconds = 15) : base(config, traceWriter)
{
_performanceManager = performanceManager;
_metricsLogger = metricsLogger;
_performanceCheckPeriodSeconds = performanceCheckPeriodSeconds;
}
public UserLogMetricsLogger(string category, IMetricsLogger metricsLogger, IExternalScopeProvider scopeProvider)
{
_category = category ?? throw new ArgumentNullException(nameof(category));
_metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
_scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider));
}
protected virtual void Initialize()
{
string hostLogPath = Path.Combine(ScriptConfig.RootLogPath, "Host");
FileUtility.EnsureDirectoryExists(hostLogPath);
string debugSentinelFileName = Path.Combine(hostLogPath, ScriptConstants.DebugSentinelFileName);
this.LastDebugNotify = File.GetLastWriteTime(debugSentinelFileName);
IMetricsLogger metricsLogger = ScriptConfig.HostConfig.GetService <IMetricsLogger>();
if (metricsLogger == null)
{
metricsLogger = new MetricsLogger();
ScriptConfig.HostConfig.AddService <IMetricsLogger>(metricsLogger);
}
using (metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
// read host.json and apply to JobHostConfiguration
string hostConfigFilePath = Path.Combine(ScriptConfig.RootScriptPath, ScriptConstants.HostMetadataFileName);
// If it doesn't exist, create an empty JSON file
if (!File.Exists(hostConfigFilePath))
{
File.WriteAllText(hostConfigFilePath, "{}");
}
if (ScriptConfig.HostConfig.IsDevelopment || InDebugMode)
{
// If we're in debug/development mode, use optimal debug settings
ScriptConfig.HostConfig.UseDevelopmentSettings();
}
string json = File.ReadAllText(hostConfigFilePath);
JObject hostConfig;
try
{
hostConfig = JObject.Parse(json);
}
catch (JsonException ex)
{
throw new FormatException(string.Format("Unable to parse {0} file.", ScriptConstants.HostMetadataFileName), ex);
}
ApplyConfiguration(hostConfig, ScriptConfig);
// Set up a host level TraceMonitor that will receive notification
// of ALL errors that occur. This allows us to inspect/log errors.
var traceMonitor = new TraceMonitor()
.Filter(p => { return(true); })
.Subscribe(HandleHostError);
ScriptConfig.HostConfig.Tracing.Tracers.Add(traceMonitor);
TraceLevel hostTraceLevel = ScriptConfig.HostConfig.Tracing.ConsoleLevel;
if (ScriptConfig.FileLoggingMode != FileLoggingMode.Never)
{
// Host file logging is only done conditionally
string hostLogFilePath = Path.Combine(ScriptConfig.RootLogPath, "Host");
TraceWriter fileTraceWriter = new FileTraceWriter(hostLogFilePath, hostTraceLevel).Conditional(p => FileLoggingEnabled);
if (TraceWriter != null)
{
// create a composite writer so our host logs are written to both
TraceWriter = new CompositeTraceWriter(new[] { TraceWriter, fileTraceWriter });
}
else
{
TraceWriter = fileTraceWriter;
}
}
if (TraceWriter != null)
{
ScriptConfig.HostConfig.Tracing.Tracers.Add(TraceWriter);
}
else
{
// if no TraceWriter has been configured, default it to Console
TraceWriter = new ConsoleTraceWriter(hostTraceLevel);
}
_debugModeFileWatcher = new AutoRecoveringFileSystemWatcher(hostLogPath, ScriptConstants.DebugSentinelFileName,
includeSubdirectories: false, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed);
_debugModeFileWatcher.Changed += OnDebugModeFileChanged;
var storageString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
Task <BlobLeaseManager> blobManagerCreation = null;
if (storageString == null)
{
// Disable core storage
ScriptConfig.HostConfig.StorageConnectionString = null;
blobManagerCreation = Task.FromResult <BlobLeaseManager>(null);
}
else
{
blobManagerCreation = BlobLeaseManager.CreateAsync(storageString, TimeSpan.FromSeconds(15), ScriptConfig.HostConfig.HostId, InstanceId, TraceWriter);
}
var bindingProviders = LoadBindingProviders(ScriptConfig, hostConfig, TraceWriter);
ScriptConfig.BindingProviders = bindingProviders;
TraceWriter.Info(string.Format(CultureInfo.InvariantCulture, "Reading host configuration file '{0}'", hostConfigFilePath));
if (ScriptConfig.FileWatchingEnabled)
{
_scriptFileWatcher = new AutoRecoveringFileSystemWatcher(ScriptConfig.RootScriptPath);
_scriptFileWatcher.Changed += OnFileChanged;
}
// If a file change should result in a restart, we debounce the event to
// ensure that only a single restart is triggered within a specific time window.
// This allows us to deal with a large set of file change events that might
// result from a bulk copy/unzip operation. In such cases, we only want to
// restart after ALL the operations are complete and there is a quiet period.
_restart = (e) =>
{
TraceWriter.Info(string.Format(CultureInfo.InvariantCulture, "File change of type '{0}' detected for '{1}'", e.ChangeType, e.FullPath));
TraceWriter.Info("Host configuration has changed. Signaling restart.");
RestartHost();
};
_restart = _restart.Debounce(500);
// take a snapshot so we can detect function additions/removals
_directoryCountSnapshot = Directory.EnumerateDirectories(ScriptConfig.RootScriptPath).Count();
List <FunctionDescriptorProvider> descriptionProviders = new List <FunctionDescriptorProvider>()
{
new ScriptFunctionDescriptorProvider(this, ScriptConfig),
new NodeFunctionDescriptorProvider(this, ScriptConfig),
new DotNetFunctionDescriptorProvider(this, ScriptConfig),
new PowerShellFunctionDescriptorProvider(this, ScriptConfig)
};
// Allow BindingProviders to initialize
foreach (var bindingProvider in ScriptConfig.BindingProviders)
{
try
{
bindingProvider.Initialize();
}
catch (Exception ex)
{
// If we're unable to initialize a binding provider for any reason, log the error
// and continue
TraceWriter.Error(string.Format("Error initializing binding provider '{0}'", bindingProvider.GetType().FullName), ex);
}
}
// Create the lease manager that will keep handle the primary host blob lease acquisition and renewal
// and subscribe for change notifications.
_blobLeaseManager = blobManagerCreation.GetAwaiter().GetResult();
if (_blobLeaseManager != null)
{
_blobLeaseManager.HasLeaseChanged += BlobLeaseManagerHasLeaseChanged;
}
// read all script functions and apply to JobHostConfiguration
Collection <FunctionDescriptor> functions = ReadFunctions(descriptionProviders);
Collection <CustomAttributeBuilder> typeAttributes = CreateTypeAttributes(ScriptConfig);
string defaultNamespace = "Host";
string typeName = string.Format(CultureInfo.InvariantCulture, "{0}.{1}", defaultNamespace, "Functions");
TraceWriter.Info(string.Format(CultureInfo.InvariantCulture, "Generating {0} job function(s)", functions.Count));
Type type = FunctionGenerator.Generate(HostAssemblyName, typeName, typeAttributes, functions);
List <Type> types = new List <Type>();
types.Add(type);
ScriptConfig.HostConfig.TypeLocator = new TypeLocator(types);
Functions = functions;
if (ScriptConfig.FileLoggingMode != FileLoggingMode.Never)
{
PurgeOldLogDirectories();
}
}
}
public WebHostRpcWorkerChannelManager(IScriptEventManager eventManager, IEnvironment environment, ILoggerFactory loggerFactory, IRpcWorkerChannelFactory rpcWorkerChannelFactory, IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, IMetricsLogger metricsLogger, IOptionsMonitor <LanguageWorkerOptions> languageWorkerOptions)
{
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
_eventManager = eventManager;
_loggerFactory = loggerFactory;
_metricsLogger = metricsLogger;
_rpcWorkerChannelFactory = rpcWorkerChannelFactory;
_logger = loggerFactory.CreateLogger <WebHostRpcWorkerChannelManager>();
_applicationHostOptions = applicationHostOptions;
_lanuageworkerOptions = languageWorkerOptions;
_shutdownStandbyWorkerChannels = ScheduleShutdownStandbyChannels;
_shutdownStandbyWorkerChannels = _shutdownStandbyWorkerChannels.Debounce(milliseconds: 5000);
}
public RpcWorkerConfigFactory(IConfiguration config, ILogger logger, ISystemRuntimeInformation systemRuntimeInfo, IEnvironment environment, IMetricsLogger metricsLogger)
{
_config = config ?? throw new ArgumentNullException(nameof(config));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_systemRuntimeInformation = systemRuntimeInfo ?? throw new ArgumentNullException(nameof(systemRuntimeInfo));
_environment = environment ?? throw new ArgumentNullException(nameof(environment));
_metricsLogger = metricsLogger;
WorkersDirPath = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.CodeBase).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName);
var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}");
if (!string.IsNullOrEmpty(workersDirectorySection.Value))
{
WorkersDirPath = workersDirectorySection.Value;
}
}
public SecretManager(ISecretsRepository repository, ILogger logger, IMetricsLogger metricsLogger, HostNameProvider hostNameProvider, bool createHostSecretsIfMissing = false)
: this(repository, new DefaultKeyValueConverterFactory(repository.IsEncryptionSupported), logger, metricsLogger, hostNameProvider, createHostSecretsIfMissing)
{
}
internal TestWorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, bool useStdErrStreamForErrorsOnly = false)
: base(eventManager, processRegistry, workerProcessLogger, consoleLogSource, metricsLogger, serviceProvider, useStdErrStreamForErrorsOnly)
{
}
public async Task Invoke(HttpContext httpContext, IOptions <HttpOptions> httpOptions, HttpRequestQueue requestQueue, HostPerformanceManager performanceManager, IMetricsLogger metricsLogger)
{
if (httpOptions.Value.DynamicThrottlesEnabled &&
((DateTime.UtcNow - _lastPerformanceCheck) > _performanceCheckInterval))
{
// only check host status periodically
Collection <string> exceededCounters = new Collection <string>();
_rejectRequests = await performanceManager.IsUnderHighLoadAsync(exceededCounters);
_lastPerformanceCheck = DateTime.UtcNow;
if (_rejectRequests)
{
_logger.LogWarning($"Thresholds for the following counters have been exceeded: [{string.Join(", ", exceededCounters)}]");
}
}
if (_rejectRequests)
{
// we're currently in reject mode, so reject the request and
// call the next delegate without calling base
RejectRequest(httpContext, metricsLogger);
return;
}
if (requestQueue.Enabled)
{
var success = await requestQueue.Post(httpContext, _next);
if (!success)
{
_logger?.LogInformation($"Http request queue limit of {httpOptions.Value.MaxOutstandingRequests} has been exceeded.");
RejectRequest(httpContext, metricsLogger);
}
}
else
{
// queue is not enabled, so just dispatch the request directly
await _next.Invoke(httpContext);
}
}
public SecretManager(ISecretsRepository repository, ILogger logger, IMetricsLogger metricsLogger, bool createHostSecretsIfMissing = false)
: this(repository, new DefaultKeyValueConverterFactory(), logger, metricsLogger, createHostSecretsIfMissing)
{
}
private static IServiceCollection AddFunctionMetadataManager(this IServiceCollection services, ScriptApplicationHostOptions options, IMetricsLogger metricsLogger)
{
var factory = new TestOptionsFactory <ScriptApplicationHostOptions>(options);
var source = new TestChangeTokenSource <ScriptApplicationHostOptions>();
var changeTokens = new[] { source };
var optionsMonitor = new OptionsMonitor <ScriptApplicationHostOptions>(factory, changeTokens, factory);
var metadataProvider = new FunctionMetadataProvider(optionsMonitor, NullLogger <FunctionMetadataProvider> .Instance, metricsLogger);
var metadataManager = TestFunctionMetadataManager.GetFunctionMetadataManager(new OptionsWrapper <ScriptJobHostOptions>(new ScriptJobHostOptions()), metadataProvider, new List <IFunctionProvider>(), new OptionsWrapper <HttpWorkerOptions>(new HttpWorkerOptions()), new NullLoggerFactory(), new OptionsWrapper <LanguageWorkerOptions>(TestHelpers.GetTestLanguageWorkerOptions()));
services.AddSingleton <IFunctionMetadataManager>(metadataManager);
services.AddSingleton <IFunctionMetadataProvider>(metadataProvider);
return(services);
}
public FunctionMetadataProvider(IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, IOptions <LanguageWorkerOptions> languageWorkerOptions, ILogger <FunctionMetadataProvider> logger, IMetricsLogger metricsLogger)
{
_applicationHostOptions = applicationHostOptions;
_metricsLogger = metricsLogger;
_logger = logger;
_workerConfigs = languageWorkerOptions.Value.WorkerConfigs;
}
public MockInvoker(ScriptHost host, IMetricsLogger metrics, FunctionMetadata metadata) : base(host, metadata)
{
_fastLogger = new FunctionInstanceLogger(
(name) => this.Host.GetFunctionOrNull(name),
metrics);
}
public WebScriptHostManager(ScriptHostConfiguration config) : base(config)
{
_metricsLogger = new WebHostMetricsLogger();
}
public DisposableEvent(string eventName, string functionName, IMetricsLogger metricsLogger)
{
_metricEvent = metricsLogger.BeginEvent(eventName, functionName, $"{{\"IsStopwatchHighResolution:\" {Stopwatch.IsHighResolution}}}");
_metricsLogger = metricsLogger;
}
// Specify the "builtin binding types". These are types that are directly accesible without needing an explicit load gesture.
// This is the set of bindings we shipped prior to binding extensibility.
// Map from BindingType to the Assembly Qualified Type name for its IExtensionConfigProvider object.
public ScriptHost(IOptions <JobHostOptions> options,
IOptions <HttpWorkerOptions> httpWorkerOptions,
IEnvironment environment,
IJobHostContextFactory jobHostContextFactory,
IConfiguration configuration,
IDistributedLockManager distributedLockManager,
IScriptEventManager eventManager,
ILoggerFactory loggerFactory,
IFunctionInvocationDispatcherFactory functionDispatcherFactory,
IFunctionMetadataManager functionMetadataManager,
IFileLoggingStatusManager fileLoggingStatusManager,
IMetricsLogger metricsLogger,
IOptions <ScriptJobHostOptions> scriptHostOptions,
ITypeLocator typeLocator,
IScriptHostManager scriptHostManager,
IDebugStateProvider debugManager,
IEnumerable <IScriptBindingProvider> bindingProviders,
IPrimaryHostStateProvider primaryHostStateProvider,
IJobHostMetadataProvider metadataProvider,
IHostIdProvider hostIdProvider,
IHttpRoutesManager httpRoutesManager,
IApplicationLifetime applicationLifetime,
IExtensionBundleManager extensionBundleManager,
ScriptSettingsManager settingsManager = null)
: base(options, jobHostContextFactory)
{
_environment = environment;
_typeLocator = typeLocator as ScriptTypeLocator
?? throw new ArgumentException(nameof(typeLocator), $"A {nameof(ScriptTypeLocator)} instance is required.");
_instanceId = Guid.NewGuid().ToString();
_hostOptions = options;
_configuration = configuration;
_storageConnectionString = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
_distributedLockManager = distributedLockManager;
_functionMetadataManager = functionMetadataManager;
_fileLoggingStatusManager = fileLoggingStatusManager;
_applicationLifetime = applicationLifetime;
_hostIdProvider = hostIdProvider;
_httpRoutesManager = httpRoutesManager;
_isHttpWorker = httpWorkerOptions.Value.Description != null;
ScriptOptions = scriptHostOptions.Value;
_scriptHostManager = scriptHostManager;
FunctionErrors = new Dictionary <string, ICollection <string> >(StringComparer.OrdinalIgnoreCase);
EventManager = eventManager;
_functionDispatcher = functionDispatcherFactory.GetFunctionDispatcher();
_settingsManager = settingsManager ?? ScriptSettingsManager.Instance;
ExtensionBundleManager = extensionBundleManager;
_metricsLogger = metricsLogger;
_hostLogPath = Path.Combine(ScriptOptions.RootLogPath, "Host");
_workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
_loggerFactory = loggerFactory;
_logger = loggerFactory.CreateLogger(LogCategories.Startup);
Logger = _logger;
_debugManager = debugManager;
_primaryHostStateProvider = primaryHostStateProvider;
_bindingProviders = new List <IScriptBindingProvider>(bindingProviders);
_metadataProvider = metadataProvider;
_eventSubscriptions.Add(EventManager.OfType <FunctionIndexingEvent>()
.Subscribe(evt =>
{
HandleHostError(evt.Exception);
}));
}
public HostJsonFileConfigurationSource(ScriptApplicationHostOptions applicationHostOptions, IEnvironment environment, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger)
{
if (loggerFactory == null)
{
throw new ArgumentNullException(nameof(loggerFactory));
}
HostOptions = applicationHostOptions;
Environment = environment;
_metricsLogger = metricsLogger;
_logger = loggerFactory.CreateLogger(LogCategories.Startup);
}
public ScriptStartupTypeLocator(string rootScriptPath, ILogger <ScriptStartupTypeLocator> logger, IExtensionBundleManager extensionBundleManager, IFunctionMetadataProvider functionMetadataProvider, IMetricsLogger metricsLogger)
{
_rootScriptPath = rootScriptPath ?? throw new ArgumentNullException(nameof(rootScriptPath));
_extensionBundleManager = extensionBundleManager ?? throw new ArgumentNullException(nameof(extensionBundleManager));
_logger = logger;
_functionMetadataProvider = functionMetadataProvider;
_metricsLogger = metricsLogger;
}
public HostJsonFileConfigurationProvider(HostJsonFileConfigurationSource configurationSource, ILogger logger, IMetricsLogger metricsLogger)
{
_configurationSource = configurationSource;
_path = new Stack <string>();
_logger = logger;
_metricsLogger = metricsLogger;
}
public static IDisposable LatencyEvent(this IMetricsLogger metricsLogger, string eventName, string functionName = null)
{
return(new DisposableEvent(eventName, functionName, metricsLogger));
}
public ILanguageWorkerChannel CreateLanguageWorkerChannel(string scriptRootPath, string language, IMetricsLogger metricsLogger, int attemptCount, bool isWebhostChannel = false, IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
{
return(new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), language, _eventManager, _testLogger, isWebhostChannel));
}
protected virtual void Initialize()
{
// read host.json and apply to JobHostConfiguration
string hostConfigFilePath = Path.Combine(ScriptConfig.RootScriptPath, HostConfigFileName);
// If it doesn't exist, create an empty JSON file
if (!File.Exists(hostConfigFilePath))
{
File.WriteAllText(hostConfigFilePath, "{}");
}
if (ScriptConfig.HostConfig.IsDevelopment)
{
ScriptConfig.HostConfig.UseDevelopmentSettings();
}
else
{
// TEMP: Until https://github.com/Azure/azure-webjobs-sdk-script/issues/100 is addressed
// we're using some presets that are a good middle ground
ScriptConfig.HostConfig.Queues.MaxPollingInterval = TimeSpan.FromSeconds(10);
ScriptConfig.HostConfig.Singleton.ListenerLockPeriod = TimeSpan.FromSeconds(15);
}
string json = File.ReadAllText(hostConfigFilePath);
JObject hostConfig = JObject.Parse(json);
ApplyConfiguration(hostConfig, ScriptConfig);
// Set up a host level TraceMonitor that will receive notificaition
// of ALL errors that occur. This allows us to inspect/log errors.
var traceMonitor = new TraceMonitor()
.Filter(p => { return(true); })
.Subscribe(HandleHostError);
ScriptConfig.HostConfig.Tracing.Tracers.Add(traceMonitor);
if (ScriptConfig.FileLoggingEnabled)
{
string hostLogFilePath = Path.Combine(ScriptConfig.RootLogPath, "Host");
TraceWriter = new FileTraceWriter(hostLogFilePath, TraceLevel.Verbose);
ScriptConfig.HostConfig.Tracing.Tracers.Add(TraceWriter);
}
else
{
TraceWriter = NullTraceWriter.Instance;
}
TraceWriter.Verbose(string.Format(CultureInfo.InvariantCulture, "Reading host configuration file '{0}'", hostConfigFilePath));
if (ScriptConfig.TraceWriter != null)
{
ScriptConfig.HostConfig.Tracing.Tracers.Add(ScriptConfig.TraceWriter);
}
else
{
ScriptConfig.TraceWriter = NullTraceWriter.Instance;
}
if (ScriptConfig.FileWatchingEnabled)
{
_fileWatcher = new FileSystemWatcher(ScriptConfig.RootScriptPath)
{
IncludeSubdirectories = true,
EnableRaisingEvents = true
};
_fileWatcher.Changed += OnFileChanged;
_fileWatcher.Created += OnFileChanged;
_fileWatcher.Deleted += OnFileChanged;
_fileWatcher.Renamed += OnFileChanged;
}
// If a file change should result in a restart, we debounce the event to
// ensure that only a single restart is triggered within a specific time window.
// This allows us to deal with a large set of file change events that might
// result from a bulk copy/unzip operation. In such cases, we only want to
// restart after ALL the operations are complete and there is a quiet period.
_restart = (e) =>
{
TraceWriter.Verbose(string.Format(CultureInfo.InvariantCulture, "File change of type '{0}' detected for '{1}'", e.ChangeType, e.FullPath));
TraceWriter.Verbose("Host configuration has changed. Signaling restart.");
// signal host restart
_restartEvent.Set();
};
_restart = _restart.Debounce(500);
// take a snapshot so we can detect function additions/removals
_directoryCountSnapshot = Directory.EnumerateDirectories(ScriptConfig.RootScriptPath).Count();
var dashboardString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Dashboard);
var config = ScriptConfig.HostConfig;
if (dashboardString != null)
{
var fastLogger = new FastLogger(dashboardString);
config.AddService <IAsyncCollector <FunctionInstanceLogEntry> >(fastLogger);
}
config.DashboardConnectionString = null; // disable slow logging
IMetricsLogger metricsLogger = ScriptConfig.HostConfig.GetService <IMetricsLogger>();
if (metricsLogger == null)
{
ScriptConfig.HostConfig.AddService <IMetricsLogger>(new MetricsLogger());
}
// Bindings may use name resolution, so provide this before reading the bindings.
var nameResolver = new NameResolver();
var storageString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
if (storageString == null)
{
// Disable core storage
ScriptConfig.HostConfig.StorageConnectionString = null;
}
ScriptConfig.HostConfig.NameResolver = nameResolver;
List <FunctionDescriptorProvider> descriptionProviders = new List <FunctionDescriptorProvider>()
{
new ScriptFunctionDescriptorProvider(this, ScriptConfig),
new NodeFunctionDescriptorProvider(this, ScriptConfig),
new CSharpFunctionDescriptionProvider(this, ScriptConfig)
};
// read all script functions and apply to JobHostConfiguration
Collection <FunctionDescriptor> functions = ReadFunctions(ScriptConfig, descriptionProviders);
string defaultNamespace = "Host";
string typeName = string.Format(CultureInfo.InvariantCulture, "{0}.{1}", defaultNamespace, "Functions");
TraceWriter.Verbose(string.Format(CultureInfo.InvariantCulture, "Generating {0} job function(s)", functions.Count));
Type type = FunctionGenerator.Generate(HostAssemblyName, typeName, functions);
List <Type> types = new List <Type>();
types.Add(type);
ScriptConfig.HostConfig.TypeLocator = new TypeLocator(types);
ApplyBindingConfiguration(functions, ScriptConfig.HostConfig);
Functions = functions;
}
internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, bool useStdErrStreamForErrorsOnly = false)
{
_processRegistry = processRegistry;
_workerProcessLogger = workerProcessLogger;
_consoleLogSource = consoleLogSource;
_eventManager = eventManager;
_metricsLogger = metricsLogger;
_useStdErrorStreamForErrorsOnly = useStdErrStreamForErrorsOnly;
_serviceProvider = serviceProvider;
// We subscribe to host start events so we can handle the restart that occurs
// on host specialization.
_eventSubscription = _eventManager.OfType <HostStartEvent>().Subscribe(OnHostStart);
}
internal FunctionInstanceLogger(IFunctionMetadataManager metadataManager, IProxyMetadataManager proxyMetadataManager, IMetricsLogger metrics)
{
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
_proxyMetadataManager = proxyMetadataManager ?? throw new ArgumentNullException(nameof(proxyMetadataManager));
_metadataManager = metadataManager ?? throw new ArgumentNullException(nameof(metadataManager));
}
internal static void LogInvocationMetrics(IMetricsLogger metrics, Collection<BindingMetadata> bindings)
{
metrics.LogEvent(MetricEventNames.FunctionInvoke);
// log events for each of the binding types used
foreach (var binding in bindings)
{
string eventName = binding.IsTrigger ?
string.Format(MetricEventNames.FunctionBindingTypeFormat, binding.Type) :
string.Format(MetricEventNames.FunctionBindingTypeDirectionFormat, binding.Type, binding.Direction);
metrics.LogEvent(eventName);
}
}