/// <summary> /// Read and apply host.json configuration. /// </summary> private JObject LoadHostConfigurationFile() { using (_metricsLogger.LatencyEvent(MetricEventNames.LoadHostConfigurationSource)) { // Before configuration has been fully read, configure a default logger factory // to ensure we can log any configuration errors. There's no filters at this point, // but that's okay since we can't build filters until we apply configuration below. // We'll recreate the loggers after config is read. We initialize the public logger // to the startup logger until we've read configuration settings and can create the real logger. // The "startup" logger is used in this class for startup related logs. The public logger is used // for all other logging after startup. ScriptApplicationHostOptions options = _configurationSource.HostOptions; string hostFilePath = Path.Combine(options.ScriptPath, ScriptConstants.HostMetadataFileName); JObject hostConfigObject = LoadHostConfig(hostFilePath); hostConfigObject = InitializeHostConfig(hostFilePath, hostConfigObject); string sanitizedJson = SanitizeHostJson(hostConfigObject); _logger.HostConfigApplied(); // Do not log these until after all the configuration is done so the proper filters are applied. _logger.HostConfigReading(hostFilePath); _logger.HostConfigRead(sanitizedJson); return(hostConfigObject); } }
public IList <RpcWorkerConfig> GetConfigs() { using (_metricsLogger.LatencyEvent(MetricEventNames.GetConfigs)) { BuildWorkerProviderDictionary(); var result = new List <RpcWorkerConfig>(); foreach (var description in _workerDescripionDictionary.Values) { _logger.LogDebug($"Worker path for language worker {description.Language}: {description.WorkerDirectory}"); var arguments = new WorkerProcessArguments() { ExecutablePath = description.DefaultExecutablePath, WorkerPath = description.DefaultWorkerPath }; arguments.ExecutableArguments.AddRange(description.Arguments); var config = new RpcWorkerConfig() { Description = description, Arguments = arguments }; result.Add(config); } return(result); } }
/// <summary> /// Performs all required initialization on the host. /// Must be called before the host is started. /// </summary> public async Task InitializeAsync() { _stopwatch.Start(); using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency)) { PreInitialize(); HostInitializing?.Invoke(this, EventArgs.Empty); // Generate Functions IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata(); _workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions); if (!_environment.IsPlaceholderModeEnabled()) { _metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, _workerRuntime)); } // Initialize language worker function dispatcher await _functionDispatcher.InitializeAsync(functions); var directTypes = GetDirectTypes(functions); await InitializeFunctionDescriptorsAsync(functions); GenerateFunctions(directTypes); CleanupFileSystem(); } }
/// <summary> /// Performs all required initialization on the host. /// Must be called before the host is started. /// </summary> public async Task InitializeAsync(CancellationToken cancellationToken = default) { _stopwatch.Start(); using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency)) { PreInitialize(); HostInitializing?.Invoke(this, EventArgs.Empty); // Generate Functions IEnumerable <FunctionMetadata> functionMetadataList = GetFunctionsMetadata(); _workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functionMetadataList); if (!_environment.IsPlaceholderModeEnabled()) { // Appending the runtime version is currently only enabled for linux consumption. // This will be eventually enabled for Windows Consumption as well. string runtimeStack = GetPreciseRuntimeStack(_workerRuntime); _metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, runtimeStack)); } var directTypes = GetDirectTypes(functionMetadataList); await InitializeFunctionDescriptorsAsync(functionMetadataList); // Initialize worker function invocation dispatcher only for valid functions after creating function descriptors await _functionDispatcher.InitializeAsync(Utility.GetValidFunctions(functionMetadataList, Functions), cancellationToken); GenerateFunctions(directTypes); CleanupFileSystem(); } }
/// <summary> /// Performs all required initialization on the host. /// Must be called before the host is started. /// </summary> public async Task InitializeAsync() { _stopwatch.Start(); if (!_environment.IsPlaceholderModeEnabled()) { string runtimeLanguage = string.IsNullOrEmpty(_workerRuntime) ? "none" : _workerRuntime; _metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, runtimeLanguage)); } using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency)) { PreInitialize(); HostInitializing?.Invoke(this, EventArgs.Empty); // Generate Functions IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata(); _workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions); if (Utility.ShouldInitializeFunctionDispatcher(_environment, functions, _workerRuntime)) { _functionDispatcher.CreateWorkerState(_workerRuntime); } var directTypes = GetDirectTypes(functions); await InitializeFunctionDescriptorsAsync(functions); GenerateFunctions(directTypes); CleanupFileSystem(); } }
public async Task SpecializeAsync() { _logger.LogInformation("Starting language worker channel specialization"); _workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); IRpcWorkerChannel rpcWorkerChannel = await GetChannelAsync(_workerRuntime); if (_workerRuntime != null && rpcWorkerChannel != null) { if (UsePlaceholderChannel(_workerRuntime)) { _logger.LogDebug("Loading environment variables for runtime: {runtime}", _workerRuntime); await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest(); } else { _logger.LogDebug("Shutting down placeholder worker. Worker is not compatible for runtime: {runtime}", _workerRuntime); // If we need to allow file edits, we should shutdown the webhost channel on specialization. await ShutdownChannelIfExistsAsync(_workerRuntime, rpcWorkerChannel.Id); } } using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels)) { _shutdownStandbyWorkerChannels(); } _logger.LogDebug("Completed language worker channel specialization"); }
internal Collection <FunctionMetadata> ReadFunctionsMetadata(IEnumerable <RpcWorkerConfig> workerConfigs, IFileSystem fileSystem = null) { _functionErrors.Clear(); fileSystem = fileSystem ?? FileUtility.Instance; using (_metricsLogger.LatencyEvent(MetricEventNames.ReadFunctionsMetadata)) { var functions = new Collection <FunctionMetadata>(); if (!fileSystem.Directory.Exists(_applicationHostOptions.CurrentValue.ScriptPath)) { return(functions); } var functionDirectories = fileSystem.Directory.EnumerateDirectories(_applicationHostOptions.CurrentValue.ScriptPath).ToImmutableArray(); foreach (var functionDirectory in functionDirectories) { var function = ReadFunctionMetadata(functionDirectory, fileSystem, workerConfigs); if (function != null) { functions.Add(function); } } return(functions); } }
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0, JobHostStartupMode startupMode = JobHostStartupMode.Normal, Guid?parentOperationId = null) { // Add this to the list of trackable startup operations. Restarts can use this to cancel any ongoing or pending operations. var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger, parentOperationId); using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartService)) { try { await _hostStartSemaphore.WaitAsync(); // Now that we're inside the semaphore, set this task as completed. This prevents // restarts from being invoked (via the PlaceholderSpecializationMiddleware) before // the IHostedService has ever started. _hostStartedSource.TrySetResult(true); await UnsynchronizedStartHostAsync(activeOperation, attemptCount, startupMode); } finally { activeOperation.Dispose(); _hostStartSemaphore.Release(); } } }
public IList <RpcWorkerConfig> GetConfigs() { using (_metricsLogger.LatencyEvent(MetricEventNames.GetConfigs)) { BuildWorkerProviderDictionary(); return(_workerDescripionDictionary.Values.ToList()); } }
/// <summary> /// Performs all required initialization on the host. /// Must be called before the host is started. /// </summary> public async Task InitializeAsync(CancellationToken cancellationToken = default) { _stopwatch = ValueStopwatch.StartNew(); using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency)) { PreInitialize(); HostInitializing?.Invoke(this, EventArgs.Empty); _workerRuntime = _workerRuntime ?? _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime); // get worker config information and check to see if worker should index or not var workerConfigs = _languageWorkerOptions.Value.WorkerConfigs; bool workerIndexing = Utility.CanWorkerIndex(workerConfigs, _environment); // Generate Functions IEnumerable <FunctionMetadata> functionMetadataList = GetFunctionsMetadata(workerIndexing); if (!_environment.IsPlaceholderModeEnabled()) { string runtimeStack = _workerRuntime; if (!string.IsNullOrEmpty(runtimeStack)) { // Appending the runtime version is currently only enabled for linux consumption. This will be eventually enabled for // Windows Consumption as well. string runtimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName); if (!string.IsNullOrEmpty(runtimeVersion)) { runtimeStack = string.Concat(runtimeStack, "-", runtimeVersion); } } _metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, Sanitizer.Sanitize(runtimeStack))); Utility.LogAutorestGeneratedJsonIfExists(ScriptOptions.RootScriptPath, _logger); } IsFunctionDataCacheEnabled = GetIsFunctionDataCacheEnabled(); await InitializeFunctionDescriptorsAsync(functionMetadataList, cancellationToken); if (!workerIndexing) { // Initialize worker function invocation dispatcher only for valid functions after creating function descriptors // Dispatcher not needed for codeless function. // Disptacher needed for non-dotnet codeless functions var filteredFunctionMetadata = functionMetadataList.Where(m => !Utility.IsCodelessDotNetLanguageFunction(m)); await _functionDispatcher.InitializeAsync(Utility.GetValidFunctions(filteredFunctionMetadata, Functions), cancellationToken); } GenerateFunctions(); ScheduleFileSystemCleanup(); } }
public async Task <string> SpecializeMSISidecar(HostAssignmentContext context) { // No cold start optimization needed for side car scenarios if (context.IsWarmupRequest) { return(null); } var msiEnabled = context.IsMSIEnabled(out var endpoint); _logger.LogInformation($"MSI enabled status: {msiEnabled}"); if (msiEnabled) { if (context.MSIContext == null) { _logger.LogWarning("Skipping specialization of MSI sidecar since MSIContext was absent"); await _meshServiceClient.NotifyHealthEvent(ContainerHealthEventType.Fatal, this.GetType(), "Could not specialize MSI sidecar since MSIContext was empty"); } else { using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationMSIInit)) { var uri = new Uri(endpoint); var address = $"http://{uri.Host}:{uri.Port}{ScriptConstants.LinuxMSISpecializationStem}"; _logger.LogDebug($"Specializing sidecar at {address}"); var requestMessage = new HttpRequestMessage(HttpMethod.Post, address) { Content = new StringContent(JsonConvert.SerializeObject(context.MSIContext), Encoding.UTF8, "application/json") }; var response = await _client.SendAsync(requestMessage); _logger.LogInformation($"Specialize MSI sidecar returned {response.StatusCode}"); if (!response.IsSuccessStatusCode) { var message = $"Specialize MSI sidecar call failed. StatusCode={response.StatusCode}"; _logger.LogError(message); await _meshServiceClient.NotifyHealthEvent(ContainerHealthEventType.Fatal, this.GetType(), "Failed to specialize MSI sidecar"); return(message); } } } } return(null); }
private async Task UnpackPackage(string filePath, string scriptPath, RunFromPackageContext pkgContext, string localSitePackagesPath) { var useLocalSitePackages = !string.IsNullOrEmpty(localSitePackagesPath); CodePackageType packageType; using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationGetPackageType)) { packageType = GetPackageType(filePath, pkgContext); } if (packageType == CodePackageType.Squashfs) { // default to mount for squashfs images if (_environment.IsMountDisabled()) { if (useLocalSitePackages) { UnsquashImage(filePath, localSitePackagesPath); await CreateBindMount(localSitePackagesPath, scriptPath); } else { UnsquashImage(filePath, scriptPath); } } else { await _meshServiceClient.MountFuse(MeshServiceClient.SquashFsOperation, filePath, scriptPath); } } else if (packageType == CodePackageType.Zip) { // default to unzip for zip packages if (_environment.IsMountEnabled()) { await _meshServiceClient.MountFuse(MeshServiceClient.ZipOperation, filePath, scriptPath); } else { if (useLocalSitePackages) { _unZipHandler.UnzipPackage(filePath, localSitePackagesPath); await CreateBindMount(localSitePackagesPath, scriptPath); } else { _unZipHandler.UnzipPackage(filePath, scriptPath); } } } }
internal void SendFunctionLoadRequestCollection(IEnumerable <FunctionMetadata> functions, ManagedDependencyOptions managedDependencyOptions) { _functionLoadRequestResponseEvent = _metricsLogger.LatencyEvent(MetricEventNames.FunctionLoadRequestResponse); FunctionLoadRequestCollection functionLoadRequestCollection = GetFunctionLoadRequestCollection(functions, managedDependencyOptions); _workerChannelLogger.LogDebug("Sending FunctionLoadRequestCollection with number of functions:'{count}'", functionLoadRequestCollection.FunctionLoadRequests.Count); // send load requests for the registered functions SendStreamingMessage(new StreamingMessage { FunctionLoadRequestCollection = functionLoadRequestCollection }); }
/// <summary> /// Performs all required initialization on the host. /// Must be called before the host is started. /// </summary> public async Task InitializeAsync(CancellationToken cancellationToken = default) { _stopwatch.Start(); using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency)) { PreInitialize(); HostInitializing?.Invoke(this, EventArgs.Empty); // Generate Functions IEnumerable <FunctionMetadata> functionMetadataList = GetFunctionsMetadata(); _workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functionMetadataList); if (!_environment.IsPlaceholderModeEnabled()) { string runtimeStack = _workerRuntime; if (!string.IsNullOrEmpty(runtimeStack)) { // Appending the runtime version is currently only enabled for linux consumption. This will be eventually enabled for // Windows Consumption as well. string runtimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName); if (!string.IsNullOrEmpty(runtimeVersion)) { runtimeStack = string.Concat(runtimeStack, "-", runtimeVersion); } } _metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, runtimeStack)); Utility.LogAutorestGeneratedJsonIfExists(ScriptOptions.RootScriptPath, _logger); } var directTypes = GetDirectTypes(functionMetadataList); await InitializeFunctionDescriptorsAsync(functionMetadataList, cancellationToken); // Initialize worker function invocation dispatcher only for valid functions after creating function descriptors // Dispatcher not needed for non-proxy codeless function. // Disptacher needed for non-dotnet codeless functions var filteredFunctionMetadata = functionMetadataList.Where(m => m.IsProxy() || !Utility.IsCodelessDotNetLanguageFunction(m)); await _functionDispatcher.InitializeAsync(Utility.GetValidFunctions(filteredFunctionMetadata, Functions), cancellationToken); GenerateFunctions(directTypes); ScheduleFileSystemCleanup(); } }
public Task StartProcessAsync() { using (_metricsLogger.LatencyEvent(MetricEventNames.ProcessStart)) { Process = CreateWorkerProcess(); try { Process.ErrorDataReceived += (sender, e) => OnErrorDataReceived(sender, e); Process.OutputDataReceived += (sender, e) => OnOutputDataReceived(sender, e); Process.Exited += (sender, e) => OnProcessExited(sender, e); Process.EnableRaisingEvents = true; _workerProcessLogger?.LogDebug($"Starting worker process with FileName:{Process.StartInfo.FileName} WorkingDirectory:{Process.StartInfo.WorkingDirectory} Arguments:{Process.StartInfo.Arguments}"); Process.Start(); _workerProcessLogger?.LogDebug($"{Process.StartInfo.FileName} process with Id={Process.Id} started"); Process.BeginErrorReadLine(); Process.BeginOutputReadLine(); // Register process only after it starts _processRegistry?.Register(Process); RegisterWithProcessMonitor(); return(Task.CompletedTask); } catch (Exception ex) { _workerProcessLogger.LogError(ex, $"Failed to start Worker Channel. Process fileName: {Process.StartInfo.FileName}"); return(Task.FromException(ex)); } } }
private async Task <MethodInfo> CreateFunctionTarget(CancellationToken cancellationToken) { try { await VerifyPackageReferencesAsync(); string eventName = string.Format(MetricEventNames.FunctionCompileLatencyByLanguageFormat, _compilationService.Language); using (_metricsLogger.LatencyEvent(eventName)) { ICompilation compilation = _compilationService.GetFunctionCompilation(Metadata); FunctionSignature functionSignature = compilation.GetEntryPointSignature(_functionEntryPointResolver); ImmutableArray <Diagnostic> bindingDiagnostics = ValidateFunctionBindingArguments(functionSignature, _triggerInputName, _inputBindings, _outputBindings, throwIfFailed: true); TraceCompilationDiagnostics(bindingDiagnostics); Assembly assembly = compilation.EmitAndLoad(cancellationToken); _assemblyLoader.CreateOrUpdateContext(Metadata, assembly, _metadataResolver, TraceWriter); // Get our function entry point _functionSignature = functionSignature; System.Reflection.TypeInfo scriptType = assembly.DefinedTypes .FirstOrDefault(t => string.Compare(t.Name, functionSignature.ParentTypeName, StringComparison.Ordinal) == 0); return(_functionEntryPointResolver.GetFunctionEntryPoint(scriptType.DeclaredMethods.ToList())); } } catch (CompilationErrorException ex) { TraceOnPrimaryHost("Function compilation error", TraceLevel.Error); TraceCompilationDiagnostics(ex.Diagnostics); throw; } }
private async Task <MethodInfo> CreateFunctionTarget(CancellationToken cancellationToken) { try { await VerifyPackageReferencesAsync(); string eventName = string.Format(MetricEventNames.FunctionCompileLatencyByLanguageFormat, _compilationService.Language); using (_metricsLogger.LatencyEvent(eventName)) { IDotNetCompilation compilation = await _compilationService.GetFunctionCompilationAsync(Metadata); Assembly assembly = compilation.Emit(cancellationToken); _assemblyLoader.CreateOrUpdateContext(Metadata, assembly, _metadataResolver, TraceWriter, Host.ScriptConfig.HostConfig.LoggerFactory); FunctionSignature functionSignature = compilation.GetEntryPointSignature(_functionEntryPointResolver); ImmutableArray <Diagnostic> bindingDiagnostics = ValidateFunctionBindingArguments(functionSignature, _triggerInputName, _inputBindings, _outputBindings, throwIfFailed: true); TraceCompilationDiagnostics(bindingDiagnostics); // Set our function entry point signature _functionSignature = functionSignature; return(_functionSignature.GetMethod(assembly)); } } catch (CompilationErrorException exc) { ImmutableArray <Diagnostic> diagnostics = AddFunctionDiagnostics(exc.Diagnostics); // Here we only need to trace to system logs TraceCompilationDiagnostics(diagnostics, LogTargets.System); throw new CompilationErrorException(exc.Message, diagnostics); } }
private async Task <string> GetToken(string tokenEndpoint) { try { using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationFetchMIToken)) { using (var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, tokenEndpoint)) { var httpClient = _httpClientFactory.CreateClient(); var msiToken = _environment.GetEnvironmentVariable(EnvironmentSettingNames.MsiSecret); httpRequestMessage.Headers.Add(ScriptConstants.XIdentityHeader, msiToken); using (var response = await httpClient.SendAsync(httpRequestMessage)) { response.EnsureSuccessStatusCode(); var readAsStringAsync = await response.Content.ReadAsStringAsync(); var msiResponse = JsonConvert.DeserializeObject <TokenServiceMsiResponse>(readAsStringAsync); return(msiResponse.AccessToken); } } } } catch (Exception e) { _logger.LogError(e, nameof(GetToken)); throw; } }
internal async Task DelayUntilWokerInitialized(CancellationToken cancellationToken) { using (_metricsLogger.LatencyEvent(MetricEventNames.DelayUntilWorkerIsInitialized)) { _workerChannelLogger.LogDebug("Initializing HttpWorker."); try { bool isWorkerReady = await _httpWorkerService.IsWorkerReady(cancellationToken); if (!isWorkerReady) { throw new TimeoutException("Initializing HttpWorker timed out."); } else { _workerChannelLogger.LogDebug("HttpWorker is Initialized."); } } catch (Exception ex) { // HttpFunctionInvocationDispatcher will handdle the worker error events _workerChannelLogger.LogError(ex, "Failed to start http worker process. workerId:{id}", Id); PublishWorkerErrorEvent(ex); throw; } } }
public Task SpecializeHostAsync() { IDisposable latencyEvent = _metricsLogger.LatencyEvent(MetricEventNames.SpecializationSpecializeHost); return(_specializationTask.Value.ContinueWith(t => { if (t.IsFaulted) { // if we fail during specialization for whatever reason // this is fatal, so we shutdown _logger.LogError(t.Exception, $"Specialization failed. Shutting down."); _applicationLifetime.StopApplication(); } latencyEvent.Dispose(); })); }
/// <summary> /// Performs all required initialization on the host. /// Must be called before the host is started. /// </summary> public async Task InitializeAsync() { _stopwatch.Start(); using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency)) { PreInitialize(); await InitializeWorkersAsync(); // Generate Functions IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata(); var directTypes = GetDirectTypes(functions); InitializeFunctionDescriptors(functions); GenerateFunctions(directTypes); CleanupFileSystem(); } }
private async Task HttpClientDownload(string filePath, Uri zipUri, bool isWarmupRequest, string token, string downloadMetricName) { HttpResponseMessage response = null; await Utility.InvokeWithRetriesAsync(async() => { try { using (_metricsLogger.LatencyEvent(downloadMetricName)) { var request = new HttpRequestMessage(HttpMethod.Get, zipUri); if (!string.IsNullOrEmpty(token)) { request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); request.Headers.Add(ScriptConstants.AzureVersionHeader, StorageBlobDownloadApiVersion); } response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); } } catch (Exception e) { const string error = "Error downloading zip content"; _logger.LogError(e, error); throw; } _logger.LogInformation( "'{response.Content.Headers.ContentLength}' bytes downloaded. IsWarmupRequest = '{isWarmupRequest}'", response.Content.Headers.ContentLength, isWarmupRequest); }, 2, TimeSpan.FromSeconds(0.5)); using (_metricsLogger.LatencyEvent(isWarmupRequest ? MetricEventNames.LinuxContainerSpecializationZipWriteWarmup : MetricEventNames.LinuxContainerSpecializationZipWrite)) { using (var content = await response.Content.ReadAsStreamAsync()) using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true)) { await content.CopyToAsync(stream); } _logger.LogInformation( "'{response.Content.Headers.ContentLength}' bytes written. IsWarmupRequest = '{isWarmupRequest}'", response.Content.Headers.ContentLength, isWarmupRequest); } }
public void UnzipPackage(string filePath, string scriptPath) { using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationZipExtract)) { _logger.LogDebug($"Extracting files to '{scriptPath}'"); ZipFile.ExtractToDirectory(filePath, scriptPath, overwriteFiles: true); _logger.LogDebug("Zip extraction complete"); } }
internal void ScheduleShutdownStandbyChannels() { using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels)) { _workerRuntime = _workerRuntime ?? _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName); if (!string.IsNullOrEmpty(_workerRuntime)) { var standbyWorkerChannels = _workerChannels.Where(ch => !ch.Key.Equals(_workerRuntime, StringComparison.InvariantCultureIgnoreCase)); foreach (var runtime in standbyWorkerChannels) { using (_metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannels, runtime.Key))) { _logger.LogInformation("Disposing standby channel for runtime:{language}", runtime.Key); if (_workerChannels.TryRemove(runtime.Key, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > standbyChannels)) { foreach (string workerId in standbyChannels.Keys) { IDisposable latencyEvent = _metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannel, workerId)); standbyChannels[workerId]?.Task.ContinueWith(channelTask => { if (channelTask.Status == TaskStatus.Faulted) { _logger.LogDebug(channelTask.Exception, "Removing errored worker channel"); } else { IRpcWorkerChannel workerChannel = channelTask.Result; if (workerChannel != null) { (channelTask.Result as IDisposable)?.Dispose(); } } latencyEvent.Dispose(); }); } } } } } } }
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext) { _logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}')"); string error = null; HttpResponseMessage response = null; try { var zipUrl = assignmentContext.ZipUrl; if (!string.IsNullOrEmpty(zipUrl)) { // make sure the zip uri is valid and accessible await Utility.InvokeWithRetriesAsync(async() => { try { using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationZipHead)) { var request = new HttpRequestMessage(HttpMethod.Head, zipUrl); response = await _client.SendAsync(request); response.EnsureSuccessStatusCode(); } } catch (Exception e) { _logger.LogError(e, $"{MetricEventNames.LinuxContainerSpecializationZipHead} failed"); throw; } }, maxRetries : 2, retryInterval : TimeSpan.FromSeconds(0.3)); // Keep this less than ~1s total } } catch (Exception e) { error = $"Invalid zip url specified (StatusCode: {response?.StatusCode})"; _logger.LogError(e, "ValidateContext failed"); } return(error); }
public Task SendFunctionEnvironmentReloadRequest() { _workerChannelLogger.LogDebug("Sending FunctionEnvironmentReloadRequest"); IDisposable latencyEvent = _metricsLogger.LatencyEvent(MetricEventNames.SpecializationEnvironmentReloadRequestResponse); _eventSubscriptions .Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.FunctionEnvironmentReloadResponse) .Timeout(workerInitTimeout) .Take(1) .Subscribe((msg) => FunctionEnvironmentReloadResponse(msg.Message.FunctionEnvironmentReloadResponse, latencyEvent), HandleWorkerEnvReloadError)); IDictionary processEnv = Environment.GetEnvironmentVariables(); FunctionEnvironmentReloadRequest request = GetFunctionEnvironmentReloadRequest(processEnv); SendStreamingMessage(new StreamingMessage { FunctionEnvironmentReloadRequest = request }); return(_reloadTask.Task); }
/// <summary> /// Performs all required initialization on the host. /// Must be called before the host is started. /// </summary> public async Task InitializeAsync() { _stopwatch.Start(); using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency)) { PreInitialize(); HostInitializing?.Invoke(this, EventArgs.Empty); // Generate Functions IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata(); if (Utility.ShouldInitiliazeLanguageWorkers(functions, _currentRuntimelanguage)) { await InitializeWorkersAsync(); } var directTypes = GetDirectTypes(functions); await InitializeFunctionDescriptorsAsync(functions); GenerateFunctions(directTypes); CleanupFileSystem(); } }
private async Task <ScriptFunc> CreateFunctionTarget(CancellationToken cancellationToken) { string eventName = string.Format(MetricEventNames.FunctionCompileLatencyByLanguageFormat, _compilationService.Language); using (_metricsLogger.LatencyEvent(eventName)) { IJavaScriptCompilation compilation = await CompileAndTraceAsync(LogTargets.System, suppressCompilationSummary : true); string functionScriptPath = compilation.Emit(cancellationToken); string script = string.Format(CultureInfo.InvariantCulture, _functionTemplate, functionScriptPath.Replace('\\', '/')); return(Edge.Func(script)); } }
public LanguageWorkerChannel( ScriptJobHostOptions scriptConfig, IScriptEventManager eventManager, IWorkerProcessFactory processFactory, IProcessRegistry processRegistry, IObservable <FunctionRegistrationContext> functionRegistrations, WorkerConfig workerConfig, Uri serverUri, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger, int attemptCount) { _workerId = Guid.NewGuid().ToString(); _scriptConfig = scriptConfig; _eventManager = eventManager; _processFactory = processFactory; _processRegistry = processRegistry; _functionRegistrations = functionRegistrations; _workerConfig = workerConfig; _serverUri = serverUri; _workerChannelLogger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}"); _userLogsConsoleLogger = loggerFactory.CreateLogger(LanguageWorkerConstants.FunctionConsoleLogCategoryName); _inboundWorkerEvents = _eventManager.OfType <InboundEvent>() .Where(msg => msg.WorkerId == _workerId); _eventSubscriptions.Add(_inboundWorkerEvents .Where(msg => msg.MessageType == MsgType.RpcLog) .Subscribe(Log)); _eventSubscriptions.Add(_eventManager.OfType <RpcEvent>() .Where(msg => msg.WorkerId == _workerId) .Subscribe(msg => { var jsonMsg = JsonConvert.SerializeObject(msg, _verboseSerializerSettings); _userLogsConsoleLogger.LogTrace(jsonMsg); })); _eventSubscriptions.Add(_eventManager.OfType <FileEvent>() .Where(msg => Config.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath))) .Throttle(TimeSpan.FromMilliseconds(300)) // debounce .Subscribe(msg => _eventManager.Publish(new HostRestartEvent()))); _startLatencyMetric = metricsLogger.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Language, attemptCount)); StartWorker(); }
public async Task <string> SpecializeMSISidecar(HostAssignmentContext context) { // No cold start optimization needed for side car scenarios if (context.IsWarmupRequest) { return(null); } string endpoint; var msiEnabled = context.IsMSIEnabled(out endpoint); _logger.LogInformation($"MSI enabled status: {msiEnabled}"); if (msiEnabled) { using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationMSIInit)) { var uri = new Uri(endpoint); var address = $"http://{uri.Host}:{uri.Port}{ScriptConstants.LinuxMSISpecializationStem}"; _logger.LogDebug($"Specializing sidecar at {address}"); var requestMessage = new HttpRequestMessage(HttpMethod.Post, address) { Content = new StringContent(JsonConvert.SerializeObject(context.MSIContext), Encoding.UTF8, "application/json") }; var response = await _client.SendAsync(requestMessage); _logger.LogInformation($"Specialize MSI sidecar returned {response.StatusCode}"); if (!response.IsSuccessStatusCode) { var message = $"Specialize MSI sidecar call failed. StatusCode={response.StatusCode}"; _logger.LogError(message); return(message); } } } return(null); }