/// <summary>
/// Read and apply host.json configuration.
/// </summary>
private JObject LoadHostConfigurationFile()
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LoadHostConfigurationSource))
{
// Before configuration has been fully read, configure a default logger factory
// to ensure we can log any configuration errors. There's no filters at this point,
// but that's okay since we can't build filters until we apply configuration below.
// We'll recreate the loggers after config is read. We initialize the public logger
// to the startup logger until we've read configuration settings and can create the real logger.
// The "startup" logger is used in this class for startup related logs. The public logger is used
// for all other logging after startup.
ScriptApplicationHostOptions options = _configurationSource.HostOptions;
string hostFilePath = Path.Combine(options.ScriptPath, ScriptConstants.HostMetadataFileName);
JObject hostConfigObject = LoadHostConfig(hostFilePath);
hostConfigObject = InitializeHostConfig(hostFilePath, hostConfigObject);
string sanitizedJson = SanitizeHostJson(hostConfigObject);
_logger.HostConfigApplied();
// Do not log these until after all the configuration is done so the proper filters are applied.
_logger.HostConfigReading(hostFilePath);
_logger.HostConfigRead(sanitizedJson);
return(hostConfigObject);
}
}
public IList <RpcWorkerConfig> GetConfigs()
{
using (_metricsLogger.LatencyEvent(MetricEventNames.GetConfigs))
{
BuildWorkerProviderDictionary();
var result = new List <RpcWorkerConfig>();
foreach (var description in _workerDescripionDictionary.Values)
{
_logger.LogDebug($"Worker path for language worker {description.Language}: {description.WorkerDirectory}");
var arguments = new WorkerProcessArguments()
{
ExecutablePath = description.DefaultExecutablePath,
WorkerPath = description.DefaultWorkerPath
};
arguments.ExecutableArguments.AddRange(description.Arguments);
var config = new RpcWorkerConfig()
{
Description = description,
Arguments = arguments
};
result.Add(config);
}
return(result);
}
}
/// <summary>
/// Performs all required initialization on the host.
/// Must be called before the host is started.
/// </summary>
public async Task InitializeAsync()
{
_stopwatch.Start();
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
PreInitialize();
HostInitializing?.Invoke(this, EventArgs.Empty);
// Generate Functions
IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata();
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions);
if (!_environment.IsPlaceholderModeEnabled())
{
_metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, _workerRuntime));
}
// Initialize language worker function dispatcher
await _functionDispatcher.InitializeAsync(functions);
var directTypes = GetDirectTypes(functions);
await InitializeFunctionDescriptorsAsync(functions);
GenerateFunctions(directTypes);
CleanupFileSystem();
}
}
/// <summary>
/// Performs all required initialization on the host.
/// Must be called before the host is started.
/// </summary>
public async Task InitializeAsync(CancellationToken cancellationToken = default)
{
_stopwatch.Start();
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
PreInitialize();
HostInitializing?.Invoke(this, EventArgs.Empty);
// Generate Functions
IEnumerable <FunctionMetadata> functionMetadataList = GetFunctionsMetadata();
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functionMetadataList);
if (!_environment.IsPlaceholderModeEnabled())
{
// Appending the runtime version is currently only enabled for linux consumption.
// This will be eventually enabled for Windows Consumption as well.
string runtimeStack = GetPreciseRuntimeStack(_workerRuntime);
_metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, runtimeStack));
}
var directTypes = GetDirectTypes(functionMetadataList);
await InitializeFunctionDescriptorsAsync(functionMetadataList);
// Initialize worker function invocation dispatcher only for valid functions after creating function descriptors
await _functionDispatcher.InitializeAsync(Utility.GetValidFunctions(functionMetadataList, Functions), cancellationToken);
GenerateFunctions(directTypes);
CleanupFileSystem();
}
}
/// <summary>
/// Performs all required initialization on the host.
/// Must be called before the host is started.
/// </summary>
public async Task InitializeAsync()
{
_stopwatch.Start();
if (!_environment.IsPlaceholderModeEnabled())
{
string runtimeLanguage = string.IsNullOrEmpty(_workerRuntime) ? "none" : _workerRuntime;
_metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, runtimeLanguage));
}
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
PreInitialize();
HostInitializing?.Invoke(this, EventArgs.Empty);
// Generate Functions
IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata();
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functions);
if (Utility.ShouldInitializeFunctionDispatcher(_environment, functions, _workerRuntime))
{
_functionDispatcher.CreateWorkerState(_workerRuntime);
}
var directTypes = GetDirectTypes(functions);
await InitializeFunctionDescriptorsAsync(functions);
GenerateFunctions(directTypes);
CleanupFileSystem();
}
}
public async Task SpecializeAsync()
{
_logger.LogInformation("Starting language worker channel specialization");
_workerRuntime = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
IRpcWorkerChannel rpcWorkerChannel = await GetChannelAsync(_workerRuntime);
if (_workerRuntime != null && rpcWorkerChannel != null)
{
if (UsePlaceholderChannel(_workerRuntime))
{
_logger.LogDebug("Loading environment variables for runtime: {runtime}", _workerRuntime);
await rpcWorkerChannel.SendFunctionEnvironmentReloadRequest();
}
else
{
_logger.LogDebug("Shutting down placeholder worker. Worker is not compatible for runtime: {runtime}", _workerRuntime);
// If we need to allow file edits, we should shutdown the webhost channel on specialization.
await ShutdownChannelIfExistsAsync(_workerRuntime, rpcWorkerChannel.Id);
}
}
using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels))
{
_shutdownStandbyWorkerChannels();
}
_logger.LogDebug("Completed language worker channel specialization");
}
internal Collection <FunctionMetadata> ReadFunctionsMetadata(IEnumerable <RpcWorkerConfig> workerConfigs, IFileSystem fileSystem = null)
{
_functionErrors.Clear();
fileSystem = fileSystem ?? FileUtility.Instance;
using (_metricsLogger.LatencyEvent(MetricEventNames.ReadFunctionsMetadata))
{
var functions = new Collection <FunctionMetadata>();
if (!fileSystem.Directory.Exists(_applicationHostOptions.CurrentValue.ScriptPath))
{
return(functions);
}
var functionDirectories = fileSystem.Directory.EnumerateDirectories(_applicationHostOptions.CurrentValue.ScriptPath).ToImmutableArray();
foreach (var functionDirectory in functionDirectories)
{
var function = ReadFunctionMetadata(functionDirectory, fileSystem, workerConfigs);
if (function != null)
{
functions.Add(function);
}
}
return(functions);
}
}
private async Task StartHostAsync(CancellationToken cancellationToken, int attemptCount = 0,
JobHostStartupMode startupMode = JobHostStartupMode.Normal, Guid?parentOperationId = null)
{
// Add this to the list of trackable startup operations. Restarts can use this to cancel any ongoing or pending operations.
var activeOperation = ScriptHostStartupOperation.Create(cancellationToken, _logger, parentOperationId);
using (_metricsLogger.LatencyEvent(MetricEventNames.ScriptHostManagerStartService))
{
try
{
await _hostStartSemaphore.WaitAsync();
// Now that we're inside the semaphore, set this task as completed. This prevents
// restarts from being invoked (via the PlaceholderSpecializationMiddleware) before
// the IHostedService has ever started.
_hostStartedSource.TrySetResult(true);
await UnsynchronizedStartHostAsync(activeOperation, attemptCount, startupMode);
}
finally
{
activeOperation.Dispose();
_hostStartSemaphore.Release();
}
}
}
public IList <RpcWorkerConfig> GetConfigs()
{
using (_metricsLogger.LatencyEvent(MetricEventNames.GetConfigs))
{
BuildWorkerProviderDictionary();
return(_workerDescripionDictionary.Values.ToList());
}
}
/// <summary>
/// Performs all required initialization on the host.
/// Must be called before the host is started.
/// </summary>
public async Task InitializeAsync(CancellationToken cancellationToken = default)
{
_stopwatch = ValueStopwatch.StartNew();
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
PreInitialize();
HostInitializing?.Invoke(this, EventArgs.Empty);
_workerRuntime = _workerRuntime ?? _environment.GetEnvironmentVariable(EnvironmentSettingNames.FunctionWorkerRuntime);
// get worker config information and check to see if worker should index or not
var workerConfigs = _languageWorkerOptions.Value.WorkerConfigs;
bool workerIndexing = Utility.CanWorkerIndex(workerConfigs, _environment);
// Generate Functions
IEnumerable <FunctionMetadata> functionMetadataList = GetFunctionsMetadata(workerIndexing);
if (!_environment.IsPlaceholderModeEnabled())
{
string runtimeStack = _workerRuntime;
if (!string.IsNullOrEmpty(runtimeStack))
{
// Appending the runtime version is currently only enabled for linux consumption. This will be eventually enabled for
// Windows Consumption as well.
string runtimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName);
if (!string.IsNullOrEmpty(runtimeVersion))
{
runtimeStack = string.Concat(runtimeStack, "-", runtimeVersion);
}
}
_metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, Sanitizer.Sanitize(runtimeStack)));
Utility.LogAutorestGeneratedJsonIfExists(ScriptOptions.RootScriptPath, _logger);
}
IsFunctionDataCacheEnabled = GetIsFunctionDataCacheEnabled();
await InitializeFunctionDescriptorsAsync(functionMetadataList, cancellationToken);
if (!workerIndexing)
{
// Initialize worker function invocation dispatcher only for valid functions after creating function descriptors
// Dispatcher not needed for codeless function.
// Disptacher needed for non-dotnet codeless functions
var filteredFunctionMetadata = functionMetadataList.Where(m => !Utility.IsCodelessDotNetLanguageFunction(m));
await _functionDispatcher.InitializeAsync(Utility.GetValidFunctions(filteredFunctionMetadata, Functions), cancellationToken);
}
GenerateFunctions();
ScheduleFileSystemCleanup();
}
}
public async Task <string> SpecializeMSISidecar(HostAssignmentContext context)
{
// No cold start optimization needed for side car scenarios
if (context.IsWarmupRequest)
{
return(null);
}
var msiEnabled = context.IsMSIEnabled(out var endpoint);
_logger.LogInformation($"MSI enabled status: {msiEnabled}");
if (msiEnabled)
{
if (context.MSIContext == null)
{
_logger.LogWarning("Skipping specialization of MSI sidecar since MSIContext was absent");
await _meshServiceClient.NotifyHealthEvent(ContainerHealthEventType.Fatal, this.GetType(),
"Could not specialize MSI sidecar since MSIContext was empty");
}
else
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationMSIInit))
{
var uri = new Uri(endpoint);
var address = $"http://{uri.Host}:{uri.Port}{ScriptConstants.LinuxMSISpecializationStem}";
_logger.LogDebug($"Specializing sidecar at {address}");
var requestMessage = new HttpRequestMessage(HttpMethod.Post, address)
{
Content = new StringContent(JsonConvert.SerializeObject(context.MSIContext),
Encoding.UTF8, "application/json")
};
var response = await _client.SendAsync(requestMessage);
_logger.LogInformation($"Specialize MSI sidecar returned {response.StatusCode}");
if (!response.IsSuccessStatusCode)
{
var message = $"Specialize MSI sidecar call failed. StatusCode={response.StatusCode}";
_logger.LogError(message);
await _meshServiceClient.NotifyHealthEvent(ContainerHealthEventType.Fatal, this.GetType(),
"Failed to specialize MSI sidecar");
return(message);
}
}
}
}
return(null);
}
private async Task UnpackPackage(string filePath, string scriptPath, RunFromPackageContext pkgContext, string localSitePackagesPath)
{
var useLocalSitePackages = !string.IsNullOrEmpty(localSitePackagesPath);
CodePackageType packageType;
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationGetPackageType))
{
packageType = GetPackageType(filePath, pkgContext);
}
if (packageType == CodePackageType.Squashfs)
{
// default to mount for squashfs images
if (_environment.IsMountDisabled())
{
if (useLocalSitePackages)
{
UnsquashImage(filePath, localSitePackagesPath);
await CreateBindMount(localSitePackagesPath, scriptPath);
}
else
{
UnsquashImage(filePath, scriptPath);
}
}
else
{
await _meshServiceClient.MountFuse(MeshServiceClient.SquashFsOperation, filePath, scriptPath);
}
}
else if (packageType == CodePackageType.Zip)
{
// default to unzip for zip packages
if (_environment.IsMountEnabled())
{
await _meshServiceClient.MountFuse(MeshServiceClient.ZipOperation, filePath, scriptPath);
}
else
{
if (useLocalSitePackages)
{
_unZipHandler.UnzipPackage(filePath, localSitePackagesPath);
await CreateBindMount(localSitePackagesPath, scriptPath);
}
else
{
_unZipHandler.UnzipPackage(filePath, scriptPath);
}
}
}
}
internal void SendFunctionLoadRequestCollection(IEnumerable <FunctionMetadata> functions, ManagedDependencyOptions managedDependencyOptions)
{
_functionLoadRequestResponseEvent = _metricsLogger.LatencyEvent(MetricEventNames.FunctionLoadRequestResponse);
FunctionLoadRequestCollection functionLoadRequestCollection = GetFunctionLoadRequestCollection(functions, managedDependencyOptions);
_workerChannelLogger.LogDebug("Sending FunctionLoadRequestCollection with number of functions:'{count}'", functionLoadRequestCollection.FunctionLoadRequests.Count);
// send load requests for the registered functions
SendStreamingMessage(new StreamingMessage
{
FunctionLoadRequestCollection = functionLoadRequestCollection
});
}
/// <summary>
/// Performs all required initialization on the host.
/// Must be called before the host is started.
/// </summary>
public async Task InitializeAsync(CancellationToken cancellationToken = default)
{
_stopwatch.Start();
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
PreInitialize();
HostInitializing?.Invoke(this, EventArgs.Empty);
// Generate Functions
IEnumerable <FunctionMetadata> functionMetadataList = GetFunctionsMetadata();
_workerRuntime = _workerRuntime ?? Utility.GetWorkerRuntime(functionMetadataList);
if (!_environment.IsPlaceholderModeEnabled())
{
string runtimeStack = _workerRuntime;
if (!string.IsNullOrEmpty(runtimeStack))
{
// Appending the runtime version is currently only enabled for linux consumption. This will be eventually enabled for
// Windows Consumption as well.
string runtimeVersion = _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeVersionSettingName);
if (!string.IsNullOrEmpty(runtimeVersion))
{
runtimeStack = string.Concat(runtimeStack, "-", runtimeVersion);
}
}
_metricsLogger.LogEvent(string.Format(MetricEventNames.HostStartupRuntimeLanguage, runtimeStack));
Utility.LogAutorestGeneratedJsonIfExists(ScriptOptions.RootScriptPath, _logger);
}
var directTypes = GetDirectTypes(functionMetadataList);
await InitializeFunctionDescriptorsAsync(functionMetadataList, cancellationToken);
// Initialize worker function invocation dispatcher only for valid functions after creating function descriptors
// Dispatcher not needed for non-proxy codeless function.
// Disptacher needed for non-dotnet codeless functions
var filteredFunctionMetadata = functionMetadataList.Where(m => m.IsProxy() || !Utility.IsCodelessDotNetLanguageFunction(m));
await _functionDispatcher.InitializeAsync(Utility.GetValidFunctions(filteredFunctionMetadata, Functions), cancellationToken);
GenerateFunctions(directTypes);
ScheduleFileSystemCleanup();
}
}
public Task StartProcessAsync()
{
using (_metricsLogger.LatencyEvent(MetricEventNames.ProcessStart))
{
Process = CreateWorkerProcess();
try
{
Process.ErrorDataReceived += (sender, e) => OnErrorDataReceived(sender, e);
Process.OutputDataReceived += (sender, e) => OnOutputDataReceived(sender, e);
Process.Exited += (sender, e) => OnProcessExited(sender, e);
Process.EnableRaisingEvents = true;
_workerProcessLogger?.LogDebug($"Starting worker process with FileName:{Process.StartInfo.FileName} WorkingDirectory:{Process.StartInfo.WorkingDirectory} Arguments:{Process.StartInfo.Arguments}");
Process.Start();
_workerProcessLogger?.LogDebug($"{Process.StartInfo.FileName} process with Id={Process.Id} started");
Process.BeginErrorReadLine();
Process.BeginOutputReadLine();
// Register process only after it starts
_processRegistry?.Register(Process);
RegisterWithProcessMonitor();
return(Task.CompletedTask);
}
catch (Exception ex)
{
_workerProcessLogger.LogError(ex, $"Failed to start Worker Channel. Process fileName: {Process.StartInfo.FileName}");
return(Task.FromException(ex));
}
}
}
private async Task <MethodInfo> CreateFunctionTarget(CancellationToken cancellationToken)
{
try
{
await VerifyPackageReferencesAsync();
string eventName = string.Format(MetricEventNames.FunctionCompileLatencyByLanguageFormat, _compilationService.Language);
using (_metricsLogger.LatencyEvent(eventName))
{
ICompilation compilation = _compilationService.GetFunctionCompilation(Metadata);
FunctionSignature functionSignature = compilation.GetEntryPointSignature(_functionEntryPointResolver);
ImmutableArray <Diagnostic> bindingDiagnostics = ValidateFunctionBindingArguments(functionSignature, _triggerInputName, _inputBindings, _outputBindings, throwIfFailed: true);
TraceCompilationDiagnostics(bindingDiagnostics);
Assembly assembly = compilation.EmitAndLoad(cancellationToken);
_assemblyLoader.CreateOrUpdateContext(Metadata, assembly, _metadataResolver, TraceWriter);
// Get our function entry point
_functionSignature = functionSignature;
System.Reflection.TypeInfo scriptType = assembly.DefinedTypes
.FirstOrDefault(t => string.Compare(t.Name, functionSignature.ParentTypeName, StringComparison.Ordinal) == 0);
return(_functionEntryPointResolver.GetFunctionEntryPoint(scriptType.DeclaredMethods.ToList()));
}
}
catch (CompilationErrorException ex)
{
TraceOnPrimaryHost("Function compilation error", TraceLevel.Error);
TraceCompilationDiagnostics(ex.Diagnostics);
throw;
}
}
private async Task <MethodInfo> CreateFunctionTarget(CancellationToken cancellationToken)
{
try
{
await VerifyPackageReferencesAsync();
string eventName = string.Format(MetricEventNames.FunctionCompileLatencyByLanguageFormat, _compilationService.Language);
using (_metricsLogger.LatencyEvent(eventName))
{
IDotNetCompilation compilation = await _compilationService.GetFunctionCompilationAsync(Metadata);
Assembly assembly = compilation.Emit(cancellationToken);
_assemblyLoader.CreateOrUpdateContext(Metadata, assembly, _metadataResolver, TraceWriter, Host.ScriptConfig.HostConfig.LoggerFactory);
FunctionSignature functionSignature = compilation.GetEntryPointSignature(_functionEntryPointResolver);
ImmutableArray <Diagnostic> bindingDiagnostics = ValidateFunctionBindingArguments(functionSignature, _triggerInputName, _inputBindings, _outputBindings, throwIfFailed: true);
TraceCompilationDiagnostics(bindingDiagnostics);
// Set our function entry point signature
_functionSignature = functionSignature;
return(_functionSignature.GetMethod(assembly));
}
}
catch (CompilationErrorException exc)
{
ImmutableArray <Diagnostic> diagnostics = AddFunctionDiagnostics(exc.Diagnostics);
// Here we only need to trace to system logs
TraceCompilationDiagnostics(diagnostics, LogTargets.System);
throw new CompilationErrorException(exc.Message, diagnostics);
}
}
private async Task <string> GetToken(string tokenEndpoint)
{
try
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationFetchMIToken))
{
using (var httpRequestMessage = new HttpRequestMessage(HttpMethod.Get, tokenEndpoint))
{
var httpClient = _httpClientFactory.CreateClient();
var msiToken = _environment.GetEnvironmentVariable(EnvironmentSettingNames.MsiSecret);
httpRequestMessage.Headers.Add(ScriptConstants.XIdentityHeader, msiToken);
using (var response = await httpClient.SendAsync(httpRequestMessage))
{
response.EnsureSuccessStatusCode();
var readAsStringAsync = await response.Content.ReadAsStringAsync();
var msiResponse = JsonConvert.DeserializeObject <TokenServiceMsiResponse>(readAsStringAsync);
return(msiResponse.AccessToken);
}
}
}
}
catch (Exception e)
{
_logger.LogError(e, nameof(GetToken));
throw;
}
}
internal async Task DelayUntilWokerInitialized(CancellationToken cancellationToken)
{
using (_metricsLogger.LatencyEvent(MetricEventNames.DelayUntilWorkerIsInitialized))
{
_workerChannelLogger.LogDebug("Initializing HttpWorker.");
try
{
bool isWorkerReady = await _httpWorkerService.IsWorkerReady(cancellationToken);
if (!isWorkerReady)
{
throw new TimeoutException("Initializing HttpWorker timed out.");
}
else
{
_workerChannelLogger.LogDebug("HttpWorker is Initialized.");
}
}
catch (Exception ex)
{
// HttpFunctionInvocationDispatcher will handdle the worker error events
_workerChannelLogger.LogError(ex, "Failed to start http worker process. workerId:{id}", Id);
PublishWorkerErrorEvent(ex);
throw;
}
}
}
public Task SpecializeHostAsync()
{
IDisposable latencyEvent = _metricsLogger.LatencyEvent(MetricEventNames.SpecializationSpecializeHost);
return(_specializationTask.Value.ContinueWith(t =>
{
if (t.IsFaulted)
{
// if we fail during specialization for whatever reason
// this is fatal, so we shutdown
_logger.LogError(t.Exception, $"Specialization failed. Shutting down.");
_applicationLifetime.StopApplication();
}
latencyEvent.Dispose();
}));
}
/// <summary>
/// Performs all required initialization on the host.
/// Must be called before the host is started.
/// </summary>
public async Task InitializeAsync()
{
_stopwatch.Start();
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
PreInitialize();
await InitializeWorkersAsync();
// Generate Functions
IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata();
var directTypes = GetDirectTypes(functions);
InitializeFunctionDescriptors(functions);
GenerateFunctions(directTypes);
CleanupFileSystem();
}
}
private async Task HttpClientDownload(string filePath, Uri zipUri, bool isWarmupRequest, string token, string downloadMetricName)
{
HttpResponseMessage response = null;
await Utility.InvokeWithRetriesAsync(async() =>
{
try
{
using (_metricsLogger.LatencyEvent(downloadMetricName))
{
var request = new HttpRequestMessage(HttpMethod.Get, zipUri);
if (!string.IsNullOrEmpty(token))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
request.Headers.Add(ScriptConstants.AzureVersionHeader, StorageBlobDownloadApiVersion);
}
response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
response.EnsureSuccessStatusCode();
}
}
catch (Exception e)
{
const string error = "Error downloading zip content";
_logger.LogError(e, error);
throw;
}
_logger.LogInformation(
"'{response.Content.Headers.ContentLength}' bytes downloaded. IsWarmupRequest = '{isWarmupRequest}'",
response.Content.Headers.ContentLength, isWarmupRequest);
}, 2, TimeSpan.FromSeconds(0.5));
using (_metricsLogger.LatencyEvent(isWarmupRequest ? MetricEventNames.LinuxContainerSpecializationZipWriteWarmup : MetricEventNames.LinuxContainerSpecializationZipWrite))
{
using (var content = await response.Content.ReadAsStreamAsync())
using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
{
await content.CopyToAsync(stream);
}
_logger.LogInformation(
"'{response.Content.Headers.ContentLength}' bytes written. IsWarmupRequest = '{isWarmupRequest}'",
response.Content.Headers.ContentLength, isWarmupRequest);
}
}
public void UnzipPackage(string filePath, string scriptPath)
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationZipExtract))
{
_logger.LogDebug($"Extracting files to '{scriptPath}'");
ZipFile.ExtractToDirectory(filePath, scriptPath, overwriteFiles: true);
_logger.LogDebug("Zip extraction complete");
}
}
internal void ScheduleShutdownStandbyChannels()
{
using (_metricsLogger.LatencyEvent(MetricEventNames.SpecializationScheduleShutdownStandbyChannels))
{
_workerRuntime = _workerRuntime ?? _environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
if (!string.IsNullOrEmpty(_workerRuntime))
{
var standbyWorkerChannels = _workerChannels.Where(ch => !ch.Key.Equals(_workerRuntime, StringComparison.InvariantCultureIgnoreCase));
foreach (var runtime in standbyWorkerChannels)
{
using (_metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannels, runtime.Key)))
{
_logger.LogInformation("Disposing standby channel for runtime:{language}", runtime.Key);
if (_workerChannels.TryRemove(runtime.Key, out Dictionary <string, TaskCompletionSource <IRpcWorkerChannel> > standbyChannels))
{
foreach (string workerId in standbyChannels.Keys)
{
IDisposable latencyEvent = _metricsLogger.LatencyEvent(string.Format(MetricEventNames.SpecializationShutdownStandbyChannel, workerId));
standbyChannels[workerId]?.Task.ContinueWith(channelTask =>
{
if (channelTask.Status == TaskStatus.Faulted)
{
_logger.LogDebug(channelTask.Exception, "Removing errored worker channel");
}
else
{
IRpcWorkerChannel workerChannel = channelTask.Result;
if (workerChannel != null)
{
(channelTask.Result as IDisposable)?.Dispose();
}
}
latencyEvent.Dispose();
});
}
}
}
}
}
}
}
public async Task <string> ValidateContext(HostAssignmentContext assignmentContext)
{
_logger.LogInformation($"Validating host assignment context (SiteId: {assignmentContext.SiteId}, SiteName: '{assignmentContext.SiteName}')");
string error = null;
HttpResponseMessage response = null;
try
{
var zipUrl = assignmentContext.ZipUrl;
if (!string.IsNullOrEmpty(zipUrl))
{
// make sure the zip uri is valid and accessible
await Utility.InvokeWithRetriesAsync(async() =>
{
try
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationZipHead))
{
var request = new HttpRequestMessage(HttpMethod.Head, zipUrl);
response = await _client.SendAsync(request);
response.EnsureSuccessStatusCode();
}
}
catch (Exception e)
{
_logger.LogError(e, $"{MetricEventNames.LinuxContainerSpecializationZipHead} failed");
throw;
}
}, maxRetries : 2, retryInterval : TimeSpan.FromSeconds(0.3)); // Keep this less than ~1s total
}
}
catch (Exception e)
{
error = $"Invalid zip url specified (StatusCode: {response?.StatusCode})";
_logger.LogError(e, "ValidateContext failed");
}
return(error);
}
public Task SendFunctionEnvironmentReloadRequest()
{
_workerChannelLogger.LogDebug("Sending FunctionEnvironmentReloadRequest");
IDisposable latencyEvent = _metricsLogger.LatencyEvent(MetricEventNames.SpecializationEnvironmentReloadRequestResponse);
_eventSubscriptions
.Add(_inboundWorkerEvents.Where(msg => msg.MessageType == MsgType.FunctionEnvironmentReloadResponse)
.Timeout(workerInitTimeout)
.Take(1)
.Subscribe((msg) => FunctionEnvironmentReloadResponse(msg.Message.FunctionEnvironmentReloadResponse, latencyEvent), HandleWorkerEnvReloadError));
IDictionary processEnv = Environment.GetEnvironmentVariables();
FunctionEnvironmentReloadRequest request = GetFunctionEnvironmentReloadRequest(processEnv);
SendStreamingMessage(new StreamingMessage
{
FunctionEnvironmentReloadRequest = request
});
return(_reloadTask.Task);
}
/// <summary>
/// Performs all required initialization on the host.
/// Must be called before the host is started.
/// </summary>
public async Task InitializeAsync()
{
_stopwatch.Start();
using (_metricsLogger.LatencyEvent(MetricEventNames.HostStartupLatency))
{
PreInitialize();
HostInitializing?.Invoke(this, EventArgs.Empty);
// Generate Functions
IEnumerable <FunctionMetadata> functions = GetFunctionsMetadata();
if (Utility.ShouldInitiliazeLanguageWorkers(functions, _currentRuntimelanguage))
{
await InitializeWorkersAsync();
}
var directTypes = GetDirectTypes(functions);
await InitializeFunctionDescriptorsAsync(functions);
GenerateFunctions(directTypes);
CleanupFileSystem();
}
}
private async Task <ScriptFunc> CreateFunctionTarget(CancellationToken cancellationToken)
{
string eventName = string.Format(MetricEventNames.FunctionCompileLatencyByLanguageFormat, _compilationService.Language);
using (_metricsLogger.LatencyEvent(eventName))
{
IJavaScriptCompilation compilation = await CompileAndTraceAsync(LogTargets.System, suppressCompilationSummary : true);
string functionScriptPath = compilation.Emit(cancellationToken);
string script = string.Format(CultureInfo.InvariantCulture, _functionTemplate, functionScriptPath.Replace('\\', '/'));
return(Edge.Func(script));
}
}
public LanguageWorkerChannel(
ScriptJobHostOptions scriptConfig,
IScriptEventManager eventManager,
IWorkerProcessFactory processFactory,
IProcessRegistry processRegistry,
IObservable <FunctionRegistrationContext> functionRegistrations,
WorkerConfig workerConfig,
Uri serverUri,
ILoggerFactory loggerFactory,
IMetricsLogger metricsLogger,
int attemptCount)
{
_workerId = Guid.NewGuid().ToString();
_scriptConfig = scriptConfig;
_eventManager = eventManager;
_processFactory = processFactory;
_processRegistry = processRegistry;
_functionRegistrations = functionRegistrations;
_workerConfig = workerConfig;
_serverUri = serverUri;
_workerChannelLogger = loggerFactory.CreateLogger($"Worker.{workerConfig.Language}.{_workerId}");
_userLogsConsoleLogger = loggerFactory.CreateLogger(LanguageWorkerConstants.FunctionConsoleLogCategoryName);
_inboundWorkerEvents = _eventManager.OfType <InboundEvent>()
.Where(msg => msg.WorkerId == _workerId);
_eventSubscriptions.Add(_inboundWorkerEvents
.Where(msg => msg.MessageType == MsgType.RpcLog)
.Subscribe(Log));
_eventSubscriptions.Add(_eventManager.OfType <RpcEvent>()
.Where(msg => msg.WorkerId == _workerId)
.Subscribe(msg =>
{
var jsonMsg = JsonConvert.SerializeObject(msg, _verboseSerializerSettings);
_userLogsConsoleLogger.LogTrace(jsonMsg);
}));
_eventSubscriptions.Add(_eventManager.OfType <FileEvent>()
.Where(msg => Config.Extensions.Contains(Path.GetExtension(msg.FileChangeArguments.FullPath)))
.Throttle(TimeSpan.FromMilliseconds(300)) // debounce
.Subscribe(msg => _eventManager.Publish(new HostRestartEvent())));
_startLatencyMetric = metricsLogger.LatencyEvent(string.Format(MetricEventNames.WorkerInitializeLatency, workerConfig.Language, attemptCount));
StartWorker();
}
public async Task <string> SpecializeMSISidecar(HostAssignmentContext context)
{
// No cold start optimization needed for side car scenarios
if (context.IsWarmupRequest)
{
return(null);
}
string endpoint;
var msiEnabled = context.IsMSIEnabled(out endpoint);
_logger.LogInformation($"MSI enabled status: {msiEnabled}");
if (msiEnabled)
{
using (_metricsLogger.LatencyEvent(MetricEventNames.LinuxContainerSpecializationMSIInit))
{
var uri = new Uri(endpoint);
var address = $"http://{uri.Host}:{uri.Port}{ScriptConstants.LinuxMSISpecializationStem}";
_logger.LogDebug($"Specializing sidecar at {address}");
var requestMessage = new HttpRequestMessage(HttpMethod.Post, address)
{
Content = new StringContent(JsonConvert.SerializeObject(context.MSIContext),
Encoding.UTF8, "application/json")
};
var response = await _client.SendAsync(requestMessage);
_logger.LogInformation($"Specialize MSI sidecar returned {response.StatusCode}");
if (!response.IsSuccessStatusCode)
{
var message = $"Specialize MSI sidecar call failed. StatusCode={response.StatusCode}";
_logger.LogError(message);
return(message);
}
}
}
return(null);
}
