private void ProcessCrashed(object sender, ActivityHostCrashedEventArgs e)
{
// the process crashed, we need to not use it anymore
ActivityHostProcess process = sender as ActivityHostProcess;
Debug.Assert(process != null, "ActivityHostProcess did not raise event correctly");
Debug.Assert(process.Busy, "When ProcessCrashed is raised busy should not be reset");
process.MarkForRemoval = true;
if (e.FailureOnSetup)
{
// the request needs to be processed again
_failedRequests.Enqueue(e.Invoker);
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrFailedRequestsPerSec);
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrFailedRequestsQueueLength);
}
// Below call is added to fix the race condition:
// When OOP host process is crashed, ProcessCrashed() method sets the process as MarkForRemoval, context switch happened at this time
// and another process has finished and started the servicing thread, which checks the above process as MarkForRemoval and disposes it.
// ProcessFinished event handler is unregistered from the process and ActivityHostProcess.HandleTransportError will not be able to raise
// process finished event, resulting inconsistent _busyHost count.
//
DecrementHostCountAndStartThreads();
}
private void InitializeActivityHostProcesses()
{
for (int i = 0; i < 1; i++)
{
ActivityHostProcess activityHostProcess = this.CreateNewActivityHostProcess();
this._hostProcesses.Add(activityHostProcess);
}
}
private void SafelyDisposeProcess(ActivityHostProcess process)
{
process.Finished -= new EventHandler(this.ProcessFinished);
process.ProcessCrashed -= new EventHandler <ActivityHostCrashedEventArgs>(this.ProcessCrashed);
process.OnProcessIdle -= new EventHandler(this.ProcessIdle);
process.Dispose();
this._hostProcesses.Remove(process);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)-1, true);
}
private void InitializeActivityHostProcesses()
{
// create the minimum number of hosts
for (int i = 0; i < MinActivityHosts; i++)
{
ActivityHostProcess process = CreateNewActivityHostProcess();
_hostProcesses.Add(process);
}
}
private ActivityHostProcess CreateNewActivityHostProcess()
{
ActivityHostProcess activityHostProcess = new ActivityHostProcess(this._configuration.ActivityProcessIdleTimeoutSec, this._configuration.LanguageMode);
activityHostProcess.ProcessCrashed += new EventHandler <ActivityHostCrashedEventArgs>(this.ProcessCrashed);
activityHostProcess.Finished += new EventHandler(this.ProcessFinished);
activityHostProcess.OnProcessIdle += new EventHandler(this.ProcessIdle);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)1, true);
return(activityHostProcess);
}
private ActivityHostProcess CreateNewActivityHostProcess()
{
ActivityHostProcess process = new ActivityHostProcess(_configuration.ActivityProcessIdleTimeoutSec, _configuration.LanguageMode);
process.ProcessCrashed += ProcessCrashed;
process.Finished += ProcessFinished;
process.OnProcessIdle += ProcessIdle;
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize);
return(process);
}
private void ProcessCrashed(object sender, ActivityHostCrashedEventArgs e)
{
ActivityHostProcess activityHostProcess = sender as ActivityHostProcess;
activityHostProcess.MarkForRemoval = true;
if (e.FailureOnSetup)
{
this._failedRequests.Enqueue(e.Invoker);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 17, (long)1, true);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 18, (long)1, true);
}
this.DecrementHostCountAndStartThreads();
}
/// <summary>
/// Unregisters all wait handles and disposes a process
/// </summary>
/// <param name="process"></param>
private void SafelyDisposeProcess(ActivityHostProcess process)
{
process.Finished -= ProcessFinished;
process.ProcessCrashed -= ProcessCrashed;
process.OnProcessIdle -= ProcessIdle;
process.Dispose();
_hostProcesses.Remove(process);
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize,
-1);
}
private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
{
if (!invoker.IsCancelled)
{
process.Busy = true;
Interlocked.Increment(ref this._busyHosts);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 16, (long)1, true);
Tuple <ActivityHostProcess, ActivityInvoker> tuple = new Tuple <ActivityHostProcess, ActivityInvoker>(process, invoker);
ThreadPool.QueueUserWorkItem(new WaitCallback(PSOutOfProcessActivityController.RunPowerShellInActivityHostWorker), tuple);
return;
}
else
{
return;
}
}
/// <summary>
/// Method called by servicing thread. This method will run the command in the
/// specified process on a separate thread
/// </summary>
/// <param name="invoker"></param>
/// <param name="process"></param>
private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
{
if (invoker.IsCancelled)
{
return;
}
process.Busy = true;
Interlocked.Increment(ref _busyHosts);
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrBusyProcessesCount);
// Adding Tuple object with HostProcess and ActivityInvoker to ThreadPool.QueueUserWorkItem
// Back reference on ActivityHostProcess in ActivityInvoker is removed
//
Tuple <ActivityHostProcess, ActivityInvoker> tupleProcessAndInvoker = new Tuple <ActivityHostProcess, ActivityInvoker>(process, invoker);
ThreadPool.QueueUserWorkItem(RunPowerShellInActivityHostWorker, tupleProcessAndInvoker);
}
/// <summary>
/// Method called by servicing thread. This method will run the command in the
/// specified process on a separate thread
/// </summary>
/// <param name="invoker"></param>
/// <param name="process"></param>
private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
{
if (invoker.IsCancelled)
{
return;
}
process.Busy = true;
Interlocked.Increment(ref _busyHosts);
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrBusyProcessesCount);
// Adding Tuple object with HostProcess and ActivityInvoker to ThreadPool.QueueUserWorkItem
// Back reference on ActivityHostProcess in ActivityInvoker is removed
//
Tuple<ActivityHostProcess, ActivityInvoker> tupleProcessAndInvoker = new Tuple<ActivityHostProcess, ActivityInvoker>(process, invoker);
ThreadPool.QueueUserWorkItem(RunPowerShellInActivityHostWorker, tupleProcessAndInvoker);
}
/// <summary>
/// Unregisters all wait handles and disposes a process
/// </summary>
/// <param name="process"></param>
private void SafelyDisposeProcess(ActivityHostProcess process)
{
process.Finished -= ProcessFinished;
process.ProcessCrashed -= ProcessCrashed;
process.OnProcessIdle -= ProcessIdle;
process.Dispose();
_hostProcesses.Remove(process);
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize,
-1);
}
private ActivityHostProcess CreateNewActivityHostProcess()
{
ActivityHostProcess process = new ActivityHostProcess(_configuration.ActivityProcessIdleTimeoutSec, _configuration.LanguageMode);
process.ProcessCrashed += ProcessCrashed;
process.Finished += ProcessFinished;
process.OnProcessIdle += ProcessIdle;
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize);
return process;
}
/// <summary>
/// Method which performs the actual servicing of requests
/// </summary>
/// <param name="state"></param>
private void ServiceRequests(object state)
{
bool isFailedRequest = false;
while (Interlocked.CompareExchange(ref _busyHosts, _configuration.MaxActivityProcesses, _configuration.MaxActivityProcesses) < _configuration.MaxActivityProcesses)
{
// if there are any processes marked for removal
// remove them
List <ActivityHostProcess> toRemove = _hostProcesses.Where(process => process.MarkForRemoval).ToList();
foreach (var process in toRemove)
{
SafelyDisposeProcess(process);
}
ActivityInvoker invoker;
// first service previously failed request
// and then a queued request
if (_failedRequests.Count > 0)
{
_failedRequests.TryDequeue(out invoker);
isFailedRequest = true;
}
else
{
_requests.TryDequeue(out invoker);
isFailedRequest = false;
}
if (invoker == null)
{
break;
}
if (invoker.IsCancelled)
{
continue;
}
if (isFailedRequest)
{
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrFailedRequestsQueueLength,
-1);
}
else
{
PerfCountersMgr.UpdateCounterByValue(
PSWorkflowPerformanceCounterSetInfo.CounterSetId,
PSWorkflowPerformanceCounterIds.ActivityHostMgrPendingRequestsQueueLength,
-1);
}
bool processed = false;
foreach (ActivityHostProcess process in _hostProcesses.Where(process => !process.Busy))
{
// we have found the first free process available use it
processed = true;
RunInProcess(invoker, process);
break;
}
// if there weren't enough processes, then we create one more
if (processed)
{
continue;
}
ActivityHostProcess hostProcess = CreateNewActivityHostProcess();
_hostProcesses.Add(hostProcess);
RunInProcess(invoker, hostProcess);
}
// we are all done, set servicing to false
Interlocked.CompareExchange(ref _isServicing, NotServicing, Servicing);
// Try to start the servicing thread again if there are any pending requests and activity host processes are available.
// This is required to fix the race condition between CheckAndStartServicingThread() and ServiceRequests() methods.
//
if ((_failedRequests.Count > 0 || _requests.Count > 0) &&
Interlocked.CompareExchange(ref _busyHosts, _configuration.MaxActivityProcesses, _configuration.MaxActivityProcesses) < _configuration.MaxActivityProcesses)
{
CheckAndStartServicingThread();
}
}
private void SafelyDisposeProcess(ActivityHostProcess process)
{
process.Finished -= new EventHandler(this.ProcessFinished);
process.ProcessCrashed -= new EventHandler<ActivityHostCrashedEventArgs>(this.ProcessCrashed);
process.OnProcessIdle -= new EventHandler(this.ProcessIdle);
process.Dispose();
this._hostProcesses.Remove(process);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)-1, true);
}
private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
{
if (!invoker.IsCancelled)
{
process.Busy = true;
Interlocked.Increment(ref this._busyHosts);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 16, (long)1, true);
Tuple<ActivityHostProcess, ActivityInvoker> tuple = new Tuple<ActivityHostProcess, ActivityInvoker>(process, invoker);
ThreadPool.QueueUserWorkItem(new WaitCallback(PSOutOfProcessActivityController.RunPowerShellInActivityHostWorker), tuple);
return;
}
else
{
return;
}
}
private ActivityHostProcess CreateNewActivityHostProcess()
{
ActivityHostProcess activityHostProcess = new ActivityHostProcess(this._configuration.ActivityProcessIdleTimeoutSec, this._configuration.LanguageMode);
activityHostProcess.ProcessCrashed += new EventHandler<ActivityHostCrashedEventArgs>(this.ProcessCrashed);
activityHostProcess.Finished += new EventHandler(this.ProcessFinished);
activityHostProcess.OnProcessIdle += new EventHandler(this.ProcessIdle);
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)1, true);
return activityHostProcess;
}
private void ServiceRequests(object state)
{
ActivityInvoker activityInvoker = null;
bool flag = false;
while (Interlocked.CompareExchange(ref this._busyHosts, this._configuration.MaxActivityProcesses, this._configuration.MaxActivityProcesses) < this._configuration.MaxActivityProcesses)
{
Collection <ActivityHostProcess> activityHostProcesses = this._hostProcesses;
List <ActivityHostProcess> list = activityHostProcesses.Where <ActivityHostProcess>((ActivityHostProcess process) => process.MarkForRemoval).ToList <ActivityHostProcess>();
foreach (ActivityHostProcess activityHostProcess in list)
{
this.SafelyDisposeProcess(activityHostProcess);
}
if (this._failedRequests.Count <= 0)
{
this._requests.TryDequeue(out activityInvoker);
flag = false;
}
else
{
this._failedRequests.TryDequeue(out activityInvoker);
flag = true;
}
if (activityInvoker == null)
{
break;
}
if (activityInvoker.IsCancelled)
{
continue;
}
if (!flag)
{
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 20, (long)-1, true);
}
else
{
PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 18, (long)-1, true);
}
bool flag1 = false;
Collection <ActivityHostProcess> activityHostProcesses1 = this._hostProcesses;
IEnumerator <ActivityHostProcess> enumerator = activityHostProcesses1.Where <ActivityHostProcess>((ActivityHostProcess process) => !process.Busy).GetEnumerator();
using (enumerator)
{
if (enumerator.MoveNext())
{
ActivityHostProcess current = enumerator.Current;
flag1 = true;
this.RunInProcess(activityInvoker, current);
}
}
if (flag1)
{
continue;
}
ActivityHostProcess activityHostProcess1 = this.CreateNewActivityHostProcess();
this._hostProcesses.Add(activityHostProcess1);
this.RunInProcess(activityInvoker, activityHostProcess1);
}
Interlocked.CompareExchange(ref this._isServicing, 0, 1);
if ((this._failedRequests.Count > 0 || this._requests.Count > 0) && Interlocked.CompareExchange(ref this._busyHosts, this._configuration.MaxActivityProcesses, this._configuration.MaxActivityProcesses) < this._configuration.MaxActivityProcesses)
{
this.CheckAndStartServicingThread();
}
}
