internal CSharpFunctionInvoker(ScriptHost host, FunctionMetadata functionMetadata,
            Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings,
            IFunctionEntryPointResolver functionEntryPointResolver, FunctionAssemblyLoader assemblyLoader)
            : base(host, functionMetadata)
        {
            _host = host;
            _functionEntryPointResolver = functionEntryPointResolver;
            _assemblyLoader = assemblyLoader;
            _metadataResolver = new FunctionMetadataResolver(functionMetadata, TraceWriter);
            _inputBindings = inputBindings;
            _outputBindings = outputBindings;
            _triggerInputName = GetTriggerInputName(functionMetadata);
            _metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();

            InitializeFileWatcherIfEnabled();
            _resultProcessor = CreateResultProcessor();

            _functionValueLoader = FunctionValueLoader.Create(CreateFunctionTarget);

            _reloadScript = ReloadScript;
            _reloadScript = _reloadScript.Debounce();

            _restorePackages = RestorePackages;
            _restorePackages = _restorePackages.Debounce();
        }
        internal DotNetFunctionInvoker(ScriptHost host, FunctionMetadata functionMetadata,
            Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings,
            IFunctionEntryPointResolver functionEntryPointResolver, FunctionAssemblyLoader assemblyLoader,
            ICompilationServiceFactory compilationServiceFactory, ITraceWriterFactory traceWriterFactory = null)
            : base(host, functionMetadata, traceWriterFactory)
        {
            _metricsLogger = Host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();
            _functionEntryPointResolver = functionEntryPointResolver;
            _assemblyLoader = assemblyLoader;
            _metadataResolver = new FunctionMetadataResolver(functionMetadata, host.ScriptConfig.BindingProviders, TraceWriter);
            _compilationService = compilationServiceFactory.CreateService(functionMetadata.ScriptType, _metadataResolver);
            _inputBindings = inputBindings;
            _outputBindings = outputBindings;
            _triggerInputName = functionMetadata.Bindings.FirstOrDefault(b => b.IsTrigger).Name;

            InitializeFileWatcher();

            _resultProcessor = CreateResultProcessor();

            _functionLoader = new FunctionLoader<MethodInfo>(CreateFunctionTarget);

            _reloadScript = ReloadScript;
            _reloadScript = _reloadScript.Debounce();

            _restorePackages = RestorePackages;
            _restorePackages = _restorePackages.Debounce();
        }
 internal ScriptFunctionInvoker(string scriptFilePath, ScriptHost host, FunctionMetadata functionMetadata, Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings)
     : base(host, functionMetadata)
 {
     _scriptFilePath = scriptFilePath;
     _inputBindings = inputBindings;
     _outputBindings = outputBindings;
     _metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();
 }
        /// <summary>
        /// Creates an instance of MetricsInvoker
        /// </summary>
        /// <param name="filesToProcess">Which files to process, Can be wildcards or explicit file names</param>
        /// <param name="rootPath">Root path where to look for binaries</param>
        /// <param name="output">The resulting output from metrics.exe</param>
        /// <param name="logger">Instance of IMetricsLogger</param>
        /// <returns>A MetricsInvoker instance </returns>
        public static MetricsInvoker Create(IEnumerable<string> filesToProcess, string rootPath, string output, IMetricsLogger logger)
        {
            string metricsExePath = Path.Combine(CodeQuality.CodeMetrics.ProgramFilesX86(), @"Microsoft Visual Studio 10.0\Team Tools\Static Analysis Tools\FxCop\metrics.exe");
            if (!File.Exists(metricsExePath))
            {
                logger.LogError("Could not locate " + metricsExePath + ". Please download Visual Studio Code Metrics PowerTool 10.0 at http://www.microsoft.com/downloads/en/details.aspx?FamilyID=edd1dfb0-b9fe-4e90-b6a6-5ed6f6f6e615");
                return null;
            }

            return new MetricsInvoker(metricsExePath, filesToProcess, rootPath, output, logger);
        }
        internal NodeFunctionInvoker(ScriptHost host, BindingMetadata trigger, FunctionMetadata functionMetadata, Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings)
            : base(host, functionMetadata)
        {
            _trigger = trigger;
            string scriptFilePath = functionMetadata.Source.Replace('\\', '/');
            _script = string.Format(CultureInfo.InvariantCulture, _functionTemplate, scriptFilePath);
            _inputBindings = inputBindings;
            _outputBindings = outputBindings;
            _metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();

            InitializeFileWatcherIfEnabled();
        }
        private MetricsInvoker(string metricsExePath, IEnumerable<string> filesToProcess, string rootPath, string output, IMetricsLogger logger)
        {
            this.metricsExePath = metricsExePath;
            this.output = output;
            this.logger = logger;

            this.Argument = string.Empty;
            foreach (var file in filesToProcess)
            {
                this.Argument += string.Format(" /f:\"{0}\"", Path.Combine(rootPath, file));
            }

            this.Argument += string.Format(" /out:\"{0}\"", output);
            this.Argument = this.Argument.Trim();
        }
        internal FunctionInvokerBase(ScriptHost host, FunctionMetadata functionMetadata, ITraceWriterFactory traceWriterFactory = null)
        {
            Host = host;
            Metadata = functionMetadata;
            _metrics = host.ScriptConfig.HostConfig.GetService<IMetricsLogger>();

            // Function file logging is only done conditionally
            traceWriterFactory = traceWriterFactory ?? new FunctionTraceWriterFactory(functionMetadata.Name, Host.ScriptConfig);
            TraceWriter traceWriter = traceWriterFactory.Create();
            FileTraceWriter = traceWriter.Conditional(t => Host.FileLoggingEnabled && (!(t.Properties?.ContainsKey(ScriptConstants.TracePropertyPrimaryHostKey) ?? false) || Host.IsPrimary));

            // The global trace writer used by the invoker will write all traces to both
            // the host trace writer as well as our file trace writer
            TraceWriter = host.TraceWriter != null ?
                new CompositeTraceWriter(new TraceWriter[] { FileTraceWriter, host.TraceWriter }) :
                FileTraceWriter;

            // Apply the function name as an event property to all traces
            var functionTraceProperties = new Dictionary<string, object>
            {
                { ScriptConstants.TracePropertyFunctionNameKey, Metadata.Name }
            };
            TraceWriter = TraceWriter.Apply(functionTraceProperties);
        }
Exemple #8
0
 public WebScriptHostManager(ScriptHostConfiguration config, SecretManager secretManager) : base(config)
 {
     _metricsLogger = new WebHostMetricsLogger();
     _secretManager = secretManager;
 }
 public FunctionMetadataProvider(IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, ILogger <FunctionMetadataProvider> logger, IMetricsLogger metricsLogger)
 {
     _applicationHostOptions = applicationHostOptions;
     _metricsLogger          = metricsLogger;
     _logger = logger;
 }
Exemple #10
0
        public SecretManager(ISecretsRepository repository, IKeyValueConverterFactory keyValueConverterFactory, ILogger logger, IMetricsLogger metricsLogger, bool createHostSecretsIfMissing = false)
        {
            _repository = repository;
            _keyValueConverterFactory   = keyValueConverterFactory;
            _repository.SecretsChanged += OnSecretsChanged;
            _logger              = logger;
            _metricsLogger       = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
            _repositoryClassName = _repository.GetType().Name.ToLower();

            if (createHostSecretsIfMissing)
            {
                // The SecretManager implementation of GetHostSecrets will
                // create a host secret if one is not present.
                GetHostSecretsAsync().GetAwaiter().GetResult();
            }
        }
Exemple #11
0
 public InstanceManager(IOptionsFactory <ScriptApplicationHostOptions> optionsFactory, HttpClient client, IScriptWebHostEnvironment webHostEnvironment,
                        IEnvironment environment, ILogger <InstanceManager> logger, IMetricsLogger metricsLogger)
 {
     _client             = client ?? throw new ArgumentNullException(nameof(client));
     _webHostEnvironment = webHostEnvironment ?? throw new ArgumentNullException(nameof(webHostEnvironment));
     _logger             = logger ?? throw new ArgumentNullException(nameof(logger));
     _metricsLogger      = metricsLogger;
     _environment        = environment ?? throw new ArgumentNullException(nameof(environment));
     _optionsFactory     = optionsFactory ?? throw new ArgumentNullException(nameof(optionsFactory));
 }
 public ILanguageWorkerChannel Create(string scriptRootPath, string language, IMetricsLogger metricsLogger, int attemptCount, IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
 {
     return(new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), language, _eventManager, _testLogger, throwOnProcessStartUp: _throwOnProcessStartUp));
 }
Exemple #13
0
 public DisposableEvent(string eventName, string functionName, IMetricsLogger metricsLogger)
 {
     _metricEvent   = metricsLogger.BeginEvent(eventName, functionName);
     _metricsLogger = metricsLogger;
 }
 public WebScriptHostManager(ScriptHostConfiguration config) : base(config)
 {
     _metricsLogger = new WebHostMetricsLogger();
 }
 public DisposableEvent(string eventName, IMetricsLogger metricsLogger)
 {
     _metricEvent = metricsLogger.BeginEvent(eventName);
     _metricsLogger = metricsLogger;
 }
        public static IHostBuilder AddWebScriptHost(this IHostBuilder builder, IServiceProvider rootServiceProvider,
                                                    IServiceScopeFactory rootScopeFactory, ScriptApplicationHostOptions webHostOptions, Action <IWebJobsBuilder> configureWebJobs = null)
        {
            ILoggerFactory       configLoggerFactory = rootServiceProvider.GetService <ILoggerFactory>();
            IDependencyValidator validator           = rootServiceProvider.GetService <IDependencyValidator>();
            IMetricsLogger       metricsLogger       = rootServiceProvider.GetService <IMetricsLogger>();

            builder.UseServiceProviderFactory(new JobHostScopedServiceProviderFactory(rootServiceProvider, rootScopeFactory, validator))
            .ConfigureServices(services =>
            {
                // register default configuration
                // must happen before the script host is added below
                services.ConfigureOptions <HttpOptionsSetup>();
                services.ConfigureOptions <CustomHttpHeadersOptionsSetup>();
                services.ConfigureOptions <HostHstsOptionsSetup>();
                services.ConfigureOptions <HostCorsOptionsSetup>();
                services.ConfigureOptions <CorsOptionsSetup>();
                services.ConfigureOptions <AppServiceOptionsSetup>();
                services.ConfigureOptions <HostEasyAuthOptionsSetup>();
            })
            .AddScriptHost(webHostOptions, configLoggerFactory, metricsLogger, webJobsBuilder =>
            {
                webJobsBuilder.AddAzureStorageCoreServices();

                configureWebJobs?.Invoke(webJobsBuilder);

                ConfigureRegisteredBuilders(webJobsBuilder, rootServiceProvider);

                webJobsBuilder.Services.AddSingleton <IHttpRoutesManager, WebScriptHostHttpRoutesManager>();
            })
            .ConfigureAppConfiguration(configurationBuilder =>
            {
                ConfigureRegisteredBuilders(configurationBuilder, rootServiceProvider);
            })
            .ConfigureLogging(loggingBuilder =>
            {
                loggingBuilder.Services.AddSingleton <ILoggerFactory, ScriptLoggerFactory>();

                loggingBuilder.AddWebJobsSystem <SystemLoggerProvider>();
                loggingBuilder.Services.AddSingleton <ILoggerProvider, AzureMonitorDiagnosticLoggerProvider>();

                ConfigureRegisteredBuilders(loggingBuilder, rootServiceProvider);
            })
            .ConfigureServices(services =>
            {
                var webHostEnvironment = rootServiceProvider.GetService <IScriptWebHostEnvironment>();
                var environment        = rootServiceProvider.GetService <IEnvironment>();

                if (FunctionsSyncManager.IsSyncTriggersEnvironment(webHostEnvironment, environment))
                {
                    services.AddSingleton <IHostedService, FunctionsSyncService>();
                }

                if (!environment.IsV2CompatibilityMode())
                {
                    new FunctionsMvcBuilder(services).AddNewtonsoftJson();
                }

                services.AddSingleton <HttpRequestQueue>();
                services.AddSingleton <IHostLifetime, JobHostHostLifetime>();
                services.AddSingleton <IWebJobsExceptionHandler, WebScriptHostExceptionHandler>();

                services.AddSingleton <DefaultScriptWebHookProvider>();
                services.TryAddSingleton <IScriptWebHookProvider>(p => p.GetService <DefaultScriptWebHookProvider>());
                services.TryAddSingleton <IWebHookProvider>(p => p.GetService <DefaultScriptWebHookProvider>());
                services.TryAddSingleton <IJobHostMiddlewarePipeline, DefaultMiddlewarePipeline>();
                if (environment.IsLinuxConsumption())
                {
                    services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, JobHostEasyAuthMiddleware>());
                }
                services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, CustomHttpHeadersMiddleware>());
                services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, HstsConfigurationMiddleware>());
                if (environment.IsLinuxConsumption())
                {
                    services.AddSingleton <ICorsMiddlewareFactory, CorsMiddlewareFactory>();
                    services.TryAddEnumerable(ServiceDescriptor.Singleton <IJobHostHttpMiddleware, JobHostCorsMiddleware>());
                }
                services.TryAddSingleton <IScaleMetricsRepository, TableStorageScaleMetricsRepository>();

                if (environment.IsWindowsAzureManagedHosting() || environment.IsLinuxAzureManagedHosting())
                {
                    // Enable breaking change analysis only when hosted in Azure
                    services.AddSingleton <IChangeAnalysisStateProvider, BlobChangeAnalysisStateProvider>();
                    services.AddSingleton <IHostedService, ChangeAnalysisService>();
                }

                // Make sure the registered IHostIdProvider is used
                IHostIdProvider provider = rootServiceProvider.GetService <IHostIdProvider>();
                if (provider != null)
                {
                    services.AddSingleton <IHostIdProvider>(provider);
                }

                services.AddSingleton <IDelegatingHandlerProvider, DefaultDelegatingHandlerProvider>();

                // Logging and diagnostics
                services.AddSingleton <IMetricsLogger>(a => new NonDisposableMetricsLogger(metricsLogger));
                services.AddSingleton <IEventCollectorProvider, FunctionInstanceLogCollectorProvider>();

                // Hosted services
                services.TryAddEnumerable(ServiceDescriptor.Singleton <IHostedService, FileMonitoringService>());

                ConfigureRegisteredBuilders(services, rootServiceProvider);
            });

            var debugStateProvider = rootServiceProvider.GetService <IDebugStateProvider>();

            if (debugStateProvider.InDebugMode)
            {
                builder.UseEnvironment(EnvironmentName.Development);
            }

            return(builder);
        }
Exemple #17
0
 public WebScriptHostRequestManager(HttpConfiguration config, HostPerformanceManager performanceManager, IMetricsLogger metricsLogger, TraceWriter traceWriter, int performanceCheckPeriodSeconds = 15) : base(config, traceWriter)
 {
     _performanceManager            = performanceManager;
     _metricsLogger                 = metricsLogger;
     _performanceCheckPeriodSeconds = performanceCheckPeriodSeconds;
 }
 public UserLogMetricsLogger(string category, IMetricsLogger metricsLogger, IExternalScopeProvider scopeProvider)
 {
     _category      = category ?? throw new ArgumentNullException(nameof(category));
     _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
     _scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider));
 }
        protected virtual void Initialize()
        {
            string hostLogPath = Path.Combine(ScriptConfig.RootLogPath, "Host");

            FileUtility.EnsureDirectoryExists(hostLogPath);
            string debugSentinelFileName = Path.Combine(hostLogPath, ScriptConstants.DebugSentinelFileName);

            this.LastDebugNotify = File.GetLastWriteTime(debugSentinelFileName);

            IMetricsLogger metricsLogger = ScriptConfig.HostConfig.GetService <IMetricsLogger>();

            if (metricsLogger == null)
            {
                metricsLogger = new MetricsLogger();
                ScriptConfig.HostConfig.AddService <IMetricsLogger>(metricsLogger);
            }

            using (metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
            {
                // read host.json and apply to JobHostConfiguration
                string hostConfigFilePath = Path.Combine(ScriptConfig.RootScriptPath, ScriptConstants.HostMetadataFileName);

                // If it doesn't exist, create an empty JSON file
                if (!File.Exists(hostConfigFilePath))
                {
                    File.WriteAllText(hostConfigFilePath, "{}");
                }

                if (ScriptConfig.HostConfig.IsDevelopment || InDebugMode)
                {
                    // If we're in debug/development mode, use optimal debug settings
                    ScriptConfig.HostConfig.UseDevelopmentSettings();
                }

                string  json = File.ReadAllText(hostConfigFilePath);
                JObject hostConfig;
                try
                {
                    hostConfig = JObject.Parse(json);
                }
                catch (JsonException ex)
                {
                    throw new FormatException(string.Format("Unable to parse {0} file.", ScriptConstants.HostMetadataFileName), ex);
                }

                ApplyConfiguration(hostConfig, ScriptConfig);

                // Set up a host level TraceMonitor that will receive notification
                // of ALL errors that occur. This allows us to inspect/log errors.
                var traceMonitor = new TraceMonitor()
                                   .Filter(p => { return(true); })
                                   .Subscribe(HandleHostError);
                ScriptConfig.HostConfig.Tracing.Tracers.Add(traceMonitor);

                TraceLevel hostTraceLevel = ScriptConfig.HostConfig.Tracing.ConsoleLevel;
                if (ScriptConfig.FileLoggingMode != FileLoggingMode.Never)
                {
                    // Host file logging is only done conditionally
                    string      hostLogFilePath = Path.Combine(ScriptConfig.RootLogPath, "Host");
                    TraceWriter fileTraceWriter = new FileTraceWriter(hostLogFilePath, hostTraceLevel).Conditional(p => FileLoggingEnabled);

                    if (TraceWriter != null)
                    {
                        // create a composite writer so our host logs are written to both
                        TraceWriter = new CompositeTraceWriter(new[] { TraceWriter, fileTraceWriter });
                    }
                    else
                    {
                        TraceWriter = fileTraceWriter;
                    }
                }

                if (TraceWriter != null)
                {
                    ScriptConfig.HostConfig.Tracing.Tracers.Add(TraceWriter);
                }
                else
                {
                    // if no TraceWriter has been configured, default it to Console
                    TraceWriter = new ConsoleTraceWriter(hostTraceLevel);
                }

                _debugModeFileWatcher = new AutoRecoveringFileSystemWatcher(hostLogPath, ScriptConstants.DebugSentinelFileName,
                                                                            includeSubdirectories: false, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed);

                _debugModeFileWatcher.Changed += OnDebugModeFileChanged;

                var storageString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);
                Task <BlobLeaseManager> blobManagerCreation = null;
                if (storageString == null)
                {
                    // Disable core storage
                    ScriptConfig.HostConfig.StorageConnectionString = null;
                    blobManagerCreation = Task.FromResult <BlobLeaseManager>(null);
                }
                else
                {
                    blobManagerCreation = BlobLeaseManager.CreateAsync(storageString, TimeSpan.FromSeconds(15), ScriptConfig.HostConfig.HostId, InstanceId, TraceWriter);
                }

                var bindingProviders = LoadBindingProviders(ScriptConfig, hostConfig, TraceWriter);
                ScriptConfig.BindingProviders = bindingProviders;

                TraceWriter.Info(string.Format(CultureInfo.InvariantCulture, "Reading host configuration file '{0}'", hostConfigFilePath));

                if (ScriptConfig.FileWatchingEnabled)
                {
                    _scriptFileWatcher = new AutoRecoveringFileSystemWatcher(ScriptConfig.RootScriptPath);

                    _scriptFileWatcher.Changed += OnFileChanged;
                }

                // If a file change should result in a restart, we debounce the event to
                // ensure that only a single restart is triggered within a specific time window.
                // This allows us to deal with a large set of file change events that might
                // result from a bulk copy/unzip operation. In such cases, we only want to
                // restart after ALL the operations are complete and there is a quiet period.
                _restart = (e) =>
                {
                    TraceWriter.Info(string.Format(CultureInfo.InvariantCulture, "File change of type '{0}' detected for '{1}'", e.ChangeType, e.FullPath));
                    TraceWriter.Info("Host configuration has changed. Signaling restart.");
                    RestartHost();
                };
                _restart = _restart.Debounce(500);

                // take a snapshot so we can detect function additions/removals
                _directoryCountSnapshot = Directory.EnumerateDirectories(ScriptConfig.RootScriptPath).Count();

                List <FunctionDescriptorProvider> descriptionProviders = new List <FunctionDescriptorProvider>()
                {
                    new ScriptFunctionDescriptorProvider(this, ScriptConfig),
                    new NodeFunctionDescriptorProvider(this, ScriptConfig),
                    new DotNetFunctionDescriptorProvider(this, ScriptConfig),
                    new PowerShellFunctionDescriptorProvider(this, ScriptConfig)
                };

                // Allow BindingProviders to initialize
                foreach (var bindingProvider in ScriptConfig.BindingProviders)
                {
                    try
                    {
                        bindingProvider.Initialize();
                    }
                    catch (Exception ex)
                    {
                        // If we're unable to initialize a binding provider for any reason, log the error
                        // and continue
                        TraceWriter.Error(string.Format("Error initializing binding provider '{0}'", bindingProvider.GetType().FullName), ex);
                    }
                }

                // Create the lease manager that will keep handle the primary host blob lease acquisition and renewal
                // and subscribe for change notifications.
                _blobLeaseManager = blobManagerCreation.GetAwaiter().GetResult();
                if (_blobLeaseManager != null)
                {
                    _blobLeaseManager.HasLeaseChanged += BlobLeaseManagerHasLeaseChanged;
                }

                // read all script functions and apply to JobHostConfiguration
                Collection <FunctionDescriptor>     functions      = ReadFunctions(descriptionProviders);
                Collection <CustomAttributeBuilder> typeAttributes = CreateTypeAttributes(ScriptConfig);
                string defaultNamespace = "Host";
                string typeName         = string.Format(CultureInfo.InvariantCulture, "{0}.{1}", defaultNamespace, "Functions");
                TraceWriter.Info(string.Format(CultureInfo.InvariantCulture, "Generating {0} job function(s)", functions.Count));
                Type        type  = FunctionGenerator.Generate(HostAssemblyName, typeName, typeAttributes, functions);
                List <Type> types = new List <Type>();
                types.Add(type);

                ScriptConfig.HostConfig.TypeLocator = new TypeLocator(types);

                Functions = functions;

                if (ScriptConfig.FileLoggingMode != FileLoggingMode.Never)
                {
                    PurgeOldLogDirectories();
                }
            }
        }
Exemple #20
0
        public WebHostRpcWorkerChannelManager(IScriptEventManager eventManager, IEnvironment environment, ILoggerFactory loggerFactory, IRpcWorkerChannelFactory rpcWorkerChannelFactory, IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, IMetricsLogger metricsLogger, IOptionsMonitor <LanguageWorkerOptions> languageWorkerOptions)
        {
            _environment             = environment ?? throw new ArgumentNullException(nameof(environment));
            _eventManager            = eventManager;
            _loggerFactory           = loggerFactory;
            _metricsLogger           = metricsLogger;
            _rpcWorkerChannelFactory = rpcWorkerChannelFactory;
            _logger = loggerFactory.CreateLogger <WebHostRpcWorkerChannelManager>();
            _applicationHostOptions = applicationHostOptions;
            _lanuageworkerOptions   = languageWorkerOptions;

            _shutdownStandbyWorkerChannels = ScheduleShutdownStandbyChannels;
            _shutdownStandbyWorkerChannels = _shutdownStandbyWorkerChannels.Debounce(milliseconds: 5000);
        }
Exemple #21
0
        public RpcWorkerConfigFactory(IConfiguration config, ILogger logger, ISystemRuntimeInformation systemRuntimeInfo, IEnvironment environment, IMetricsLogger metricsLogger)
        {
            _config = config ?? throw new ArgumentNullException(nameof(config));
            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
            _systemRuntimeInformation = systemRuntimeInfo ?? throw new ArgumentNullException(nameof(systemRuntimeInfo));
            _environment   = environment ?? throw new ArgumentNullException(nameof(environment));
            _metricsLogger = metricsLogger;
            WorkersDirPath = Path.Combine(Path.GetDirectoryName(new Uri(typeof(RpcWorkerConfigFactory).Assembly.CodeBase).LocalPath), RpcWorkerConstants.DefaultWorkersDirectoryName);
            var workersDirectorySection = _config.GetSection($"{RpcWorkerConstants.LanguageWorkersSectionName}:{WorkerConstants.WorkersDirectorySectionName}");

            if (!string.IsNullOrEmpty(workersDirectorySection.Value))
            {
                WorkersDirPath = workersDirectorySection.Value;
            }
        }
 public SecretManager(ISecretsRepository repository, ILogger logger, IMetricsLogger metricsLogger, HostNameProvider hostNameProvider, bool createHostSecretsIfMissing = false)
     : this(repository, new DefaultKeyValueConverterFactory(repository.IsEncryptionSupported), logger, metricsLogger, hostNameProvider, createHostSecretsIfMissing)
 {
 }
 internal TestWorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, bool useStdErrStreamForErrorsOnly = false)
     : base(eventManager, processRegistry, workerProcessLogger, consoleLogSource, metricsLogger, serviceProvider, useStdErrStreamForErrorsOnly)
 {
 }
Exemple #24
0
        public async Task Invoke(HttpContext httpContext, IOptions <HttpOptions> httpOptions, HttpRequestQueue requestQueue, HostPerformanceManager performanceManager, IMetricsLogger metricsLogger)
        {
            if (httpOptions.Value.DynamicThrottlesEnabled &&
                ((DateTime.UtcNow - _lastPerformanceCheck) > _performanceCheckInterval))
            {
                // only check host status periodically
                Collection <string> exceededCounters = new Collection <string>();
                _rejectRequests = await performanceManager.IsUnderHighLoadAsync(exceededCounters);

                _lastPerformanceCheck = DateTime.UtcNow;
                if (_rejectRequests)
                {
                    _logger.LogWarning($"Thresholds for the following counters have been exceeded: [{string.Join(", ", exceededCounters)}]");
                }
            }

            if (_rejectRequests)
            {
                // we're currently in reject mode, so reject the request and
                // call the next delegate without calling base
                RejectRequest(httpContext, metricsLogger);
                return;
            }

            if (requestQueue.Enabled)
            {
                var success = await requestQueue.Post(httpContext, _next);

                if (!success)
                {
                    _logger?.LogInformation($"Http request queue limit of {httpOptions.Value.MaxOutstandingRequests} has been exceeded.");
                    RejectRequest(httpContext, metricsLogger);
                }
            }
            else
            {
                // queue is not enabled, so just dispatch the request directly
                await _next.Invoke(httpContext);
            }
        }
Exemple #25
0
 public SecretManager(ISecretsRepository repository, ILogger logger, IMetricsLogger metricsLogger, bool createHostSecretsIfMissing = false)
     : this(repository, new DefaultKeyValueConverterFactory(), logger, metricsLogger, createHostSecretsIfMissing)
 {
 }
        private static IServiceCollection AddFunctionMetadataManager(this IServiceCollection services, ScriptApplicationHostOptions options, IMetricsLogger metricsLogger)
        {
            var factory        = new TestOptionsFactory <ScriptApplicationHostOptions>(options);
            var source         = new TestChangeTokenSource <ScriptApplicationHostOptions>();
            var changeTokens   = new[] { source };
            var optionsMonitor = new OptionsMonitor <ScriptApplicationHostOptions>(factory, changeTokens, factory);

            var metadataProvider = new FunctionMetadataProvider(optionsMonitor, NullLogger <FunctionMetadataProvider> .Instance, metricsLogger);
            var metadataManager  = TestFunctionMetadataManager.GetFunctionMetadataManager(new OptionsWrapper <ScriptJobHostOptions>(new ScriptJobHostOptions()), metadataProvider, new List <IFunctionProvider>(), new OptionsWrapper <HttpWorkerOptions>(new HttpWorkerOptions()), new NullLoggerFactory(), new OptionsWrapper <LanguageWorkerOptions>(TestHelpers.GetTestLanguageWorkerOptions()));

            services.AddSingleton <IFunctionMetadataManager>(metadataManager);
            services.AddSingleton <IFunctionMetadataProvider>(metadataProvider);

            return(services);
        }
 public FunctionMetadataProvider(IOptionsMonitor <ScriptApplicationHostOptions> applicationHostOptions, IOptions <LanguageWorkerOptions> languageWorkerOptions, ILogger <FunctionMetadataProvider> logger, IMetricsLogger metricsLogger)
 {
     _applicationHostOptions = applicationHostOptions;
     _metricsLogger          = metricsLogger;
     _logger        = logger;
     _workerConfigs = languageWorkerOptions.Value.WorkerConfigs;
 }
Exemple #28
0
 public MockInvoker(ScriptHost host, IMetricsLogger metrics, FunctionMetadata metadata) : base(host, metadata)
 {
     _fastLogger = new FunctionInstanceLogger(
         (name) => this.Host.GetFunctionOrNull(name),
         metrics);
 }
Exemple #29
0
 public WebScriptHostManager(ScriptHostConfiguration config) : base(config)
 {
     _metricsLogger = new WebHostMetricsLogger();
 }
Exemple #30
0
 public DisposableEvent(string eventName, string functionName, IMetricsLogger metricsLogger)
 {
     _metricEvent   = metricsLogger.BeginEvent(eventName, functionName, $"{{\"IsStopwatchHighResolution:\" {Stopwatch.IsHighResolution}}}");
     _metricsLogger = metricsLogger;
 }
        // Specify the "builtin binding types". These are types that are directly accesible without needing an explicit load gesture.
        // This is the set of bindings we shipped prior to binding extensibility.
        // Map from BindingType to the Assembly Qualified Type name for its IExtensionConfigProvider object.

        public ScriptHost(IOptions <JobHostOptions> options,
                          IOptions <HttpWorkerOptions> httpWorkerOptions,
                          IEnvironment environment,
                          IJobHostContextFactory jobHostContextFactory,
                          IConfiguration configuration,
                          IDistributedLockManager distributedLockManager,
                          IScriptEventManager eventManager,
                          ILoggerFactory loggerFactory,
                          IFunctionInvocationDispatcherFactory functionDispatcherFactory,
                          IFunctionMetadataManager functionMetadataManager,
                          IFileLoggingStatusManager fileLoggingStatusManager,
                          IMetricsLogger metricsLogger,
                          IOptions <ScriptJobHostOptions> scriptHostOptions,
                          ITypeLocator typeLocator,
                          IScriptHostManager scriptHostManager,
                          IDebugStateProvider debugManager,
                          IEnumerable <IScriptBindingProvider> bindingProviders,
                          IPrimaryHostStateProvider primaryHostStateProvider,
                          IJobHostMetadataProvider metadataProvider,
                          IHostIdProvider hostIdProvider,
                          IHttpRoutesManager httpRoutesManager,
                          IApplicationLifetime applicationLifetime,
                          IExtensionBundleManager extensionBundleManager,
                          ScriptSettingsManager settingsManager = null)
            : base(options, jobHostContextFactory)
        {
            _environment = environment;
            _typeLocator = typeLocator as ScriptTypeLocator
                           ?? throw new ArgumentException(nameof(typeLocator), $"A {nameof(ScriptTypeLocator)} instance is required.");

            _instanceId               = Guid.NewGuid().ToString();
            _hostOptions              = options;
            _configuration            = configuration;
            _storageConnectionString  = configuration.GetWebJobsConnectionString(ConnectionStringNames.Storage);
            _distributedLockManager   = distributedLockManager;
            _functionMetadataManager  = functionMetadataManager;
            _fileLoggingStatusManager = fileLoggingStatusManager;
            _applicationLifetime      = applicationLifetime;
            _hostIdProvider           = hostIdProvider;
            _httpRoutesManager        = httpRoutesManager;
            _isHttpWorker             = httpWorkerOptions.Value.Description != null;
            ScriptOptions             = scriptHostOptions.Value;
            _scriptHostManager        = scriptHostManager;
            FunctionErrors            = new Dictionary <string, ICollection <string> >(StringComparer.OrdinalIgnoreCase);
            EventManager              = eventManager;
            _functionDispatcher       = functionDispatcherFactory.GetFunctionDispatcher();
            _settingsManager          = settingsManager ?? ScriptSettingsManager.Instance;
            ExtensionBundleManager    = extensionBundleManager;

            _metricsLogger = metricsLogger;

            _hostLogPath = Path.Combine(ScriptOptions.RootLogPath, "Host");

            _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);

            _loggerFactory = loggerFactory;
            _logger        = loggerFactory.CreateLogger(LogCategories.Startup);
            Logger         = _logger;

            _debugManager             = debugManager;
            _primaryHostStateProvider = primaryHostStateProvider;
            _bindingProviders         = new List <IScriptBindingProvider>(bindingProviders);
            _metadataProvider         = metadataProvider;
            _eventSubscriptions.Add(EventManager.OfType <FunctionIndexingEvent>()
                                    .Subscribe(evt =>
            {
                HandleHostError(evt.Exception);
            }));
        }
Exemple #32
0
        public HostJsonFileConfigurationSource(ScriptApplicationHostOptions applicationHostOptions, IEnvironment environment, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger)
        {
            if (loggerFactory == null)
            {
                throw new ArgumentNullException(nameof(loggerFactory));
            }

            HostOptions    = applicationHostOptions;
            Environment    = environment;
            _metricsLogger = metricsLogger;
            _logger        = loggerFactory.CreateLogger(LogCategories.Startup);
        }
 public ScriptStartupTypeLocator(string rootScriptPath, ILogger <ScriptStartupTypeLocator> logger, IExtensionBundleManager extensionBundleManager, IFunctionMetadataProvider functionMetadataProvider, IMetricsLogger metricsLogger)
 {
     _rootScriptPath         = rootScriptPath ?? throw new ArgumentNullException(nameof(rootScriptPath));
     _extensionBundleManager = extensionBundleManager ?? throw new ArgumentNullException(nameof(extensionBundleManager));
     _logger = logger;
     _functionMetadataProvider = functionMetadataProvider;
     _metricsLogger            = metricsLogger;
 }
Exemple #34
0
 public HostJsonFileConfigurationProvider(HostJsonFileConfigurationSource configurationSource, ILogger logger, IMetricsLogger metricsLogger)
 {
     _configurationSource = configurationSource;
     _path          = new Stack <string>();
     _logger        = logger;
     _metricsLogger = metricsLogger;
 }
Exemple #35
0
 public static IDisposable LatencyEvent(this IMetricsLogger metricsLogger, string eventName, string functionName = null)
 {
     return(new DisposableEvent(eventName, functionName, metricsLogger));
 }
 public ILanguageWorkerChannel CreateLanguageWorkerChannel(string scriptRootPath, string language, IMetricsLogger metricsLogger, int attemptCount, bool isWebhostChannel = false, IOptions <ManagedDependencyOptions> managedDependencyOptions = null)
 {
     return(new TestLanguageWorkerChannel(Guid.NewGuid().ToString(), language, _eventManager, _testLogger, isWebhostChannel));
 }
Exemple #37
0
        protected virtual void Initialize()
        {
            // read host.json and apply to JobHostConfiguration
            string hostConfigFilePath = Path.Combine(ScriptConfig.RootScriptPath, HostConfigFileName);

            // If it doesn't exist, create an empty JSON file
            if (!File.Exists(hostConfigFilePath))
            {
                File.WriteAllText(hostConfigFilePath, "{}");
            }

            if (ScriptConfig.HostConfig.IsDevelopment)
            {
                ScriptConfig.HostConfig.UseDevelopmentSettings();
            }
            else
            {
                // TEMP: Until https://github.com/Azure/azure-webjobs-sdk-script/issues/100 is addressed
                // we're using some presets that are a good middle ground
                ScriptConfig.HostConfig.Queues.MaxPollingInterval    = TimeSpan.FromSeconds(10);
                ScriptConfig.HostConfig.Singleton.ListenerLockPeriod = TimeSpan.FromSeconds(15);
            }

            string  json       = File.ReadAllText(hostConfigFilePath);
            JObject hostConfig = JObject.Parse(json);

            ApplyConfiguration(hostConfig, ScriptConfig);

            // Set up a host level TraceMonitor that will receive notificaition
            // of ALL errors that occur. This allows us to inspect/log errors.
            var traceMonitor = new TraceMonitor()
                               .Filter(p => { return(true); })
                               .Subscribe(HandleHostError);

            ScriptConfig.HostConfig.Tracing.Tracers.Add(traceMonitor);

            if (ScriptConfig.FileLoggingEnabled)
            {
                string hostLogFilePath = Path.Combine(ScriptConfig.RootLogPath, "Host");
                TraceWriter = new FileTraceWriter(hostLogFilePath, TraceLevel.Verbose);
                ScriptConfig.HostConfig.Tracing.Tracers.Add(TraceWriter);
            }
            else
            {
                TraceWriter = NullTraceWriter.Instance;
            }

            TraceWriter.Verbose(string.Format(CultureInfo.InvariantCulture, "Reading host configuration file '{0}'", hostConfigFilePath));

            if (ScriptConfig.TraceWriter != null)
            {
                ScriptConfig.HostConfig.Tracing.Tracers.Add(ScriptConfig.TraceWriter);
            }
            else
            {
                ScriptConfig.TraceWriter = NullTraceWriter.Instance;
            }

            if (ScriptConfig.FileWatchingEnabled)
            {
                _fileWatcher = new FileSystemWatcher(ScriptConfig.RootScriptPath)
                {
                    IncludeSubdirectories = true,
                    EnableRaisingEvents   = true
                };
                _fileWatcher.Changed += OnFileChanged;
                _fileWatcher.Created += OnFileChanged;
                _fileWatcher.Deleted += OnFileChanged;
                _fileWatcher.Renamed += OnFileChanged;
            }

            // If a file change should result in a restart, we debounce the event to
            // ensure that only a single restart is triggered within a specific time window.
            // This allows us to deal with a large set of file change events that might
            // result from a bulk copy/unzip operation. In such cases, we only want to
            // restart after ALL the operations are complete and there is a quiet period.
            _restart = (e) =>
            {
                TraceWriter.Verbose(string.Format(CultureInfo.InvariantCulture, "File change of type '{0}' detected for '{1}'", e.ChangeType, e.FullPath));
                TraceWriter.Verbose("Host configuration has changed. Signaling restart.");

                // signal host restart
                _restartEvent.Set();
            };
            _restart = _restart.Debounce(500);

            // take a snapshot so we can detect function additions/removals
            _directoryCountSnapshot = Directory.EnumerateDirectories(ScriptConfig.RootScriptPath).Count();

            var dashboardString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Dashboard);

            var config = ScriptConfig.HostConfig;

            if (dashboardString != null)
            {
                var fastLogger = new FastLogger(dashboardString);
                config.AddService <IAsyncCollector <FunctionInstanceLogEntry> >(fastLogger);
            }
            config.DashboardConnectionString = null; // disable slow logging

            IMetricsLogger metricsLogger = ScriptConfig.HostConfig.GetService <IMetricsLogger>();

            if (metricsLogger == null)
            {
                ScriptConfig.HostConfig.AddService <IMetricsLogger>(new MetricsLogger());
            }

            // Bindings may use name resolution, so provide this before reading the bindings.
            var nameResolver = new NameResolver();

            var storageString = AmbientConnectionStringProvider.Instance.GetConnectionString(ConnectionStringNames.Storage);

            if (storageString == null)
            {
                // Disable core storage
                ScriptConfig.HostConfig.StorageConnectionString = null;
            }

            ScriptConfig.HostConfig.NameResolver = nameResolver;

            List <FunctionDescriptorProvider> descriptionProviders = new List <FunctionDescriptorProvider>()
            {
                new ScriptFunctionDescriptorProvider(this, ScriptConfig),
                new NodeFunctionDescriptorProvider(this, ScriptConfig),
                new CSharpFunctionDescriptionProvider(this, ScriptConfig)
            };

            // read all script functions and apply to JobHostConfiguration
            Collection <FunctionDescriptor> functions = ReadFunctions(ScriptConfig, descriptionProviders);
            string defaultNamespace = "Host";
            string typeName         = string.Format(CultureInfo.InvariantCulture, "{0}.{1}", defaultNamespace, "Functions");

            TraceWriter.Verbose(string.Format(CultureInfo.InvariantCulture, "Generating {0} job function(s)", functions.Count));
            Type        type  = FunctionGenerator.Generate(HostAssemblyName, typeName, functions);
            List <Type> types = new List <Type>();

            types.Add(type);

            ScriptConfig.HostConfig.TypeLocator = new TypeLocator(types);

            ApplyBindingConfiguration(functions, ScriptConfig.HostConfig);

            Functions = functions;
        }
Exemple #38
0
        internal WorkerProcess(IScriptEventManager eventManager, IProcessRegistry processRegistry, ILogger workerProcessLogger, IWorkerConsoleLogSource consoleLogSource, IMetricsLogger metricsLogger, IServiceProvider serviceProvider, bool useStdErrStreamForErrorsOnly = false)
        {
            _processRegistry                = processRegistry;
            _workerProcessLogger            = workerProcessLogger;
            _consoleLogSource               = consoleLogSource;
            _eventManager                   = eventManager;
            _metricsLogger                  = metricsLogger;
            _useStdErrorStreamForErrorsOnly = useStdErrStreamForErrorsOnly;
            _serviceProvider                = serviceProvider;

            // We subscribe to host start events so we can handle the restart that occurs
            // on host specialization.
            _eventSubscription = _eventManager.OfType <HostStartEvent>().Subscribe(OnHostStart);
        }
 internal FunctionInstanceLogger(IFunctionMetadataManager metadataManager, IProxyMetadataManager proxyMetadataManager, IMetricsLogger metrics)
 {
     _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
     _proxyMetadataManager = proxyMetadataManager ?? throw new ArgumentNullException(nameof(proxyMetadataManager));
     _metadataManager      = metadataManager ?? throw new ArgumentNullException(nameof(metadataManager));
 }
        internal static void LogInvocationMetrics(IMetricsLogger metrics, Collection<BindingMetadata> bindings)
        {
            metrics.LogEvent(MetricEventNames.FunctionInvoke);

            // log events for each of the binding types used
            foreach (var binding in bindings)
            {
                string eventName = binding.IsTrigger ?
                    string.Format(MetricEventNames.FunctionBindingTypeFormat, binding.Type) :
                    string.Format(MetricEventNames.FunctionBindingTypeDirectionFormat, binding.Type, binding.Direction);
                metrics.LogEvent(eventName);
            }
        }