private RpcLogger CreateLoggerWithContext(string requestId, string invocationId)
{
var logger = new RpcLogger(_msgStream);
logger.SetContext(requestId, invocationId);
return(logger);
}
internal StreamingMessage ProcessFunctionEnvironmentReloadRequest(StreamingMessage request)
{
var stopwatch = new Stopwatch();
stopwatch.Start();
var environmentReloadRequest = request.FunctionEnvironmentReloadRequest;
var rpcLogger = new RpcLogger(_msgStream);
rpcLogger.SetContext(request.RequestId, null);
var functionsEnvironmentReloader = new FunctionsEnvironmentReloader(rpcLogger);
functionsEnvironmentReloader.ReloadEnvironment(
environmentReloadRequest.EnvironmentVariables,
environmentReloadRequest.FunctionAppDirectory);
rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.EnvironmentReloadCompleted, stopwatch.ElapsedMilliseconds));
StreamingMessage response = NewStreamingMessageTemplate(
request.RequestId,
StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadResponse,
out StatusResult status);
return(response);
}
/// <summary>
/// Checkout an idle PowerShellManager instance in a non-blocking asynchronous way.
/// </summary>
internal PowerShellManager CheckoutIdleWorker(StreamingMessage request, AzFunctionInfo functionInfo)
{
PowerShellManager psManager = null;
string requestId = request.RequestId;
string invocationId = request.InvocationRequest?.InvocationId;
// If the pool has an idle one, just use it.
if (!_pool.TryTake(out psManager))
{
// The pool doesn't have an idle one.
if (_poolSize < _upperBound &&
Interlocked.Increment(ref _poolSize) <= _upperBound)
{
// If the pool hasn't reached its bounded capacity yet, then
// we create a new item and return it.
var logger = new RpcLogger(_msgStream);
logger.SetContext(requestId, invocationId);
psManager = new PowerShellManager(logger);
}
else
{
// If the pool has reached its bounded capacity, then the thread
// should be blocked until an idle one becomes available.
psManager = _pool.Take();
}
}
// Register the function with the Runspace before returning the idle PowerShellManager.
FunctionMetadata.RegisterFunctionMetadata(psManager.InstanceId, functionInfo);
psManager.Logger.SetContext(requestId, invocationId);
return(psManager);
}
/// <summary>
/// Processes the dependency download request
/// </summary>
/// <param name="msgStream">The protobuf messaging stream</param>
/// <param name="request">The StreamingMessage request for function load</param>
/// <param name="pwsh">The PowerShell instance used to download modules</param>
internal void ProcessDependencyDownload(MessagingStream msgStream, StreamingMessage request, PowerShell pwsh)
{
if (request.FunctionLoadRequest.ManagedDependencyEnabled)
{
var rpcLogger = new RpcLogger(msgStream);
rpcLogger.SetContext(request.RequestId, null);
if (!_shouldUpdateFunctionAppDependencies)
{
if (!string.IsNullOrEmpty(_dependenciesNotUpdatedMessage))
{
// We were not able to update the function app dependencies.
// However, there is a previous installation, so continue with the function app execution.
rpcLogger.Log(LogLevel.Warning, _dependenciesNotUpdatedMessage, isUserLog: true);
}
else
{
// The function app already has the latest dependencies installed.
rpcLogger.Log(LogLevel.Trace, PowerShellWorkerStrings.LatestFunctionAppDependenciesAlreadyInstalled, isUserLog: true);
}
return;
}
if (Dependencies.Count == 0)
{
// If there are no dependencies to install, log and return.
rpcLogger.Log(LogLevel.Trace, PowerShellWorkerStrings.FunctionAppDoesNotHaveDependentModulesToInstall, isUserLog: true);
return;
}
//Start dependency download on a separate thread
_dependencyDownloadTask = Task.Run(() => InstallFunctionAppDependencies(pwsh, rpcLogger));
}
}
/// <summary>
/// Method to process a InvocationRequest.
/// This method checks out a worker from the pool, and then starts the actual invocation in a threadpool thread.
/// </summary>
internal StreamingMessage ProcessInvocationRequest(StreamingMessage request)
{
Exception error = null;
try
{
if (_dependencyManager.DependencyDownloadTask != null &&
!_dependencyManager.DependencyDownloadTask.IsCompleted)
{
var rpcLogger = new RpcLogger(_msgStream);
rpcLogger.SetContext(request.RequestId, request.InvocationRequest?.InvocationId);
rpcLogger.Log(LogLevel.Information, PowerShellWorkerStrings.DependencyDownloadInProgress, isUserLog: true);
_dependencyManager.WaitOnDependencyDownload();
}
if (_dependencyManager.DependencyError != null)
{
error = _dependencyManager.DependencyError;
}
else
{
AzFunctionInfo functionInfo = FunctionLoader.GetFunctionInfo(request.InvocationRequest.FunctionId);
PowerShellManager psManager = _powershellPool.CheckoutIdleWorker(request, functionInfo);
if (_powershellPool.UpperBound == 1)
{
// When the concurrency upper bound is 1, we can handle only one invocation at a time anyways,
// so it's better to just do it on the current thread to reduce the required synchronization.
ProcessInvocationRequestImpl(request, functionInfo, psManager);
}
else
{
// When the concurrency upper bound is more than 1, we have to handle the invocation in a worker
// thread, so multiple invocations can make progress at the same time, even though by time-sharing.
Task.Run(() => ProcessInvocationRequestImpl(request, functionInfo, psManager));
}
}
}
catch (Exception e)
{
error = e;
}
if (error != null)
{
StreamingMessage response = NewStreamingMessageTemplate(
request.RequestId,
StreamingMessage.ContentOneofCase.InvocationResponse,
out StatusResult status);
response.InvocationResponse.InvocationId = request.InvocationRequest.InvocationId;
status.Status = StatusResult.Types.Status.Failure;
status.Exception = error.ToRpcException();
return(response);
}
return(null);
}
/// <summary>
/// Initialize the pool and populate it with PowerShellManager instances.
/// We instantiate PowerShellManager instances in a lazy way, starting from size 1 and increase the number of workers as needed.
/// </summary>
internal void Initialize(string requestId)
{
var logger = new RpcLogger(_msgStream);
try
{
logger.SetContext(requestId, invocationId: null);
_pool.Add(new PowerShellManager(logger));
_poolSize = 1;
}
finally
{
logger.ResetContext();
}
}
/// <summary>
/// Method to process a InvocationRequest.
/// This method checks out a worker from the pool, and then starts the actual invocation in a threadpool thread.
/// </summary>
internal StreamingMessage ProcessInvocationRequest(StreamingMessage request)
{
try
{
var stopwatch = new FunctionInvocationPerformanceStopwatch();
stopwatch.OnStart();
// Will block if installing dependencies is required
_dependencyManager.WaitForDependenciesAvailability(
() =>
{
var rpcLogger = new RpcLogger(_msgStream);
rpcLogger.SetContext(request.RequestId, request.InvocationRequest?.InvocationId);
return(rpcLogger);
});
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.DependenciesAvailable);
AzFunctionInfo functionInfo = FunctionLoader.GetFunctionInfo(request.InvocationRequest.FunctionId);
PowerShellManager psManager = _powershellPool.CheckoutIdleWorker(
request.RequestId,
request.InvocationRequest?.InvocationId,
functionInfo.FuncName,
functionInfo.OutputBindings);
stopwatch.OnCheckpoint(FunctionInvocationPerformanceStopwatch.Checkpoint.RunspaceAvailable);
// When the concurrency upper bound is more than 1, we have to handle the invocation in a worker
// thread, so multiple invocations can make progress at the same time, even though by time-sharing.
Task.Run(() => ProcessInvocationRequestImpl(request, functionInfo, psManager, stopwatch));
}
catch (Exception e)
{
StreamingMessage response = NewStreamingMessageTemplate(
request.RequestId,
StreamingMessage.ContentOneofCase.InvocationResponse,
out StatusResult status);
response.InvocationResponse.InvocationId = request.InvocationRequest.InvocationId;
status.Status = StatusResult.Types.Status.Failure;
status.Exception = e.ToRpcException();
return(response);
}
return(null);
}
/// <summary>
/// Method to process a InvocationRequest.
/// This method checks out a worker from the pool, and then starts the actual invocation in a threadpool thread.
/// </summary>
internal StreamingMessage ProcessInvocationRequest(StreamingMessage request)
{
try
{
// Will block if installing dependencies is required
_dependencyManager.WaitForDependenciesAvailability(
() =>
{
var rpcLogger = new RpcLogger(_msgStream);
rpcLogger.SetContext(request.RequestId, request.InvocationRequest?.InvocationId);
return(rpcLogger);
});
AzFunctionInfo functionInfo = FunctionLoader.GetFunctionInfo(request.InvocationRequest.FunctionId);
PowerShellManager psManager = _powershellPool.CheckoutIdleWorker(request, functionInfo);
if (_powershellPool.UpperBound == 1)
{
// When the concurrency upper bound is 1, we can handle only one invocation at a time anyways,
// so it's better to just do it on the current thread to reduce the required synchronization.
ProcessInvocationRequestImpl(request, functionInfo, psManager);
}
else
{
// When the concurrency upper bound is more than 1, we have to handle the invocation in a worker
// thread, so multiple invocations can make progress at the same time, even though by time-sharing.
Task.Run(() => ProcessInvocationRequestImpl(request, functionInfo, psManager));
}
}
catch (Exception e)
{
StreamingMessage response = NewStreamingMessageTemplate(
request.RequestId,
StreamingMessage.ContentOneofCase.InvocationResponse,
out StatusResult status);
response.InvocationResponse.InvocationId = request.InvocationRequest.InvocationId;
status.Status = StatusResult.Types.Status.Failure;
status.Exception = e.ToRpcException();
return(response);
}
return(null);
}
/// <summary>
/// Checkout an idle PowerShellManager instance in a non-blocking asynchronous way.
/// </summary>
internal PowerShellManager CheckoutIdleWorker(StreamingMessage request, AzFunctionInfo functionInfo)
{
PowerShellManager psManager = null;
string requestId = request.RequestId;
string invocationId = request.InvocationRequest?.InvocationId;
// If the pool has an idle one, just use it.
if (!_pool.TryTake(out psManager))
{
// The pool doesn't have an idle one.
if (_poolSize < _upperBound)
{
int id = Interlocked.Increment(ref _poolSize);
if (id <= _upperBound)
{
// If the pool hasn't reached its bounded capacity yet, then
// we create a new item and return it.
var logger = new RpcLogger(_msgStream);
logger.SetContext(requestId, invocationId);
psManager = new PowerShellManager(logger, id);
RpcLogger.WriteSystemLog(LogLevel.Trace, string.Format(PowerShellWorkerStrings.LogNewPowerShellManagerCreated, id.ToString()));
}
}
if (psManager == null)
{
// If the pool has reached its bounded capacity, then the thread
// should be blocked until an idle one becomes available.
psManager = _pool.Take();
}
}
// Finish the initialization if not yet.
// This applies only to the very first PowerShellManager instance, whose initialization was deferred.
psManager.Initialize();
// Register the function with the Runspace before returning the idle PowerShellManager.
FunctionMetadata.RegisterFunctionMetadata(psManager.InstanceId, functionInfo);
psManager.Logger.SetContext(requestId, invocationId);
return(psManager);
}
/// <summary>
/// Processes the dependency download request
/// </summary>
/// <param name="msgStream">The protobuf messaging stream</param>
/// <param name="request">The StreamingMessage request for function load</param>
/// <param name="pwsh">The PowerShell instance used to download modules</param>
internal void ProcessDependencyDownload(MessagingStream msgStream, StreamingMessage request, PowerShell pwsh)
{
if (request.FunctionLoadRequest.ManagedDependencyEnabled)
{
var rpcLogger = new RpcLogger(msgStream);
rpcLogger.SetContext(request.RequestId, null);
if (Dependencies.Count == 0)
{
// If there are no dependencies to install, log and return.
rpcLogger.Log(LogLevel.Trace, PowerShellWorkerStrings.FunctionAppDoesNotHaveDependentModulesToInstall, isUserLog: true);
return;
}
if (!_shouldUpdateFunctionAppDependencies)
{
// The function app already has the latest dependencies installed.
rpcLogger.Log(LogLevel.Trace, PowerShellWorkerStrings.LatestFunctionAppDependenciesAlreadyInstalled, isUserLog: true);
return;
}
//Start dependency download on a separate thread
_dependencyDownloadTask = Task.Run(() => InstallFunctionAppDependencies(pwsh, rpcLogger));
}
}
/// <summary>
/// Method to process a FunctionLoadRequest.
/// FunctionLoadRequest should be processed sequentially. There is no point to process FunctionLoadRequest
/// concurrently as a FunctionApp doesn't include a lot functions in general. Having this step sequential
/// will make the Runspace-level initialization easier and more predictable.
/// </summary>
internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request)
{
FunctionLoadRequest functionLoadRequest = request.FunctionLoadRequest;
StreamingMessage response = NewStreamingMessageTemplate(
request.RequestId,
StreamingMessage.ContentOneofCase.FunctionLoadResponse,
out StatusResult status);
response.FunctionLoadResponse.FunctionId = functionLoadRequest.FunctionId;
// The worker may occasionally receive multiple function load requests with
// the same FunctionId. In order to make function load request idempotent,
// the worker should ignore the duplicates.
if (FunctionLoader.IsLoaded(functionLoadRequest.FunctionId))
{
// If FunctionLoader considers this function loaded, this means
// the previous request was successful, so respond accordingly.
return(response);
}
// When a functionLoadRequest comes in, we check to see if a dependency download has failed in a previous call
// or if PowerShell could not be initialized. If this is the case, mark this as a failed request
// and submit the exception to the Host (runtime).
if (_initTerminatingError != null)
{
status.Status = StatusResult.Types.Status.Failure;
status.Exception = _initTerminatingError.ToRpcException();
return(response);
}
// Ideally, the initialization should happen when processing 'WorkerInitRequest', however, the 'WorkerInitRequest'
// message doesn't provide information about the FunctionApp. That information is not available until the first
// 'FunctionLoadRequest' comes in. Therefore, we run initialization here.
// Also, we receive a FunctionLoadRequest when a proxy is configured. Proxies don't have the Metadata.Directory set
// which would cause initialization issues with the PSModulePath. Since they don't have that set, we skip over them.
if (!_isFunctionAppInitialized && !functionLoadRequest.Metadata.IsProxy)
{
try
{
_isFunctionAppInitialized = true;
var rpcLogger = new RpcLogger(_msgStream);
rpcLogger.SetContext(request.RequestId, null);
_dependencyManager = new DependencyManager(request.FunctionLoadRequest.Metadata.Directory);
var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger);
// Setup the FunctionApp root path and module path.
FunctionLoader.SetupWellKnownPaths(functionLoadRequest, managedDependenciesPath);
// Create the very first Runspace so the debugger has the target to attach to.
// This PowerShell instance is shared by the first PowerShellManager instance created in the pool,
// and the dependency manager (used to download dependent modules if needed).
var pwsh = Utils.NewPwshInstance();
_powershellPool.Initialize(pwsh);
// Start the download asynchronously if needed.
_dependencyManager.StartDependencyInstallationIfNeeded(request, pwsh, rpcLogger);
}
catch (Exception e)
{
// Failure that happens during this step is terminating and we will need to return a failure response to
// all subsequent 'FunctionLoadRequest'. Cache the exception so we can reuse it in future calls.
_initTerminatingError = e;
status.Status = StatusResult.Types.Status.Failure;
status.Exception = e.ToRpcException();
return(response);
}
}
try
{
// Load the metadata of the function.
FunctionLoader.LoadFunction(functionLoadRequest);
}
catch (Exception e)
{
status.Status = StatusResult.Types.Status.Failure;
status.Exception = e.ToRpcException();
}
return(response);
}