private void ProcessCrashed(object sender, ActivityHostCrashedEventArgs e)
        {
            // the process crashed, we need to not use it anymore
            ActivityHostProcess process = sender as ActivityHostProcess;

            Debug.Assert(process != null, "ActivityHostProcess did not raise event correctly");
            Debug.Assert(process.Busy, "When ProcessCrashed is raised busy should not be reset");
            process.MarkForRemoval = true;

            if (e.FailureOnSetup)
            {
                // the request needs to be processed again
                _failedRequests.Enqueue(e.Invoker);
                PerfCountersMgr.UpdateCounterByValue(
                    PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                    PSWorkflowPerformanceCounterIds.ActivityHostMgrFailedRequestsPerSec);
                PerfCountersMgr.UpdateCounterByValue(
                    PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                    PSWorkflowPerformanceCounterIds.ActivityHostMgrFailedRequestsQueueLength);
            }

            // Below call is added to fix the race condition:
            // When OOP host process is crashed, ProcessCrashed() method sets the process as MarkForRemoval, context switch happened at this time
            // and another process has finished and started the servicing thread, which checks the above process as MarkForRemoval and disposes it.
            // ProcessFinished event handler is unregistered from the process and ActivityHostProcess.HandleTransportError will not be able to raise
            // process finished event, resulting inconsistent _busyHost count.
            //
            DecrementHostCountAndStartThreads();
        }
예제 #2
0
 private void InitializeActivityHostProcesses()
 {
     for (int i = 0; i < 1; i++)
     {
         ActivityHostProcess activityHostProcess = this.CreateNewActivityHostProcess();
         this._hostProcesses.Add(activityHostProcess);
     }
 }
예제 #3
0
 private void SafelyDisposeProcess(ActivityHostProcess process)
 {
     process.Finished       -= new EventHandler(this.ProcessFinished);
     process.ProcessCrashed -= new EventHandler <ActivityHostCrashedEventArgs>(this.ProcessCrashed);
     process.OnProcessIdle  -= new EventHandler(this.ProcessIdle);
     process.Dispose();
     this._hostProcesses.Remove(process);
     PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)-1, true);
 }
 private void InitializeActivityHostProcesses()
 {
     // create the minimum number of hosts
     for (int i = 0; i < MinActivityHosts; i++)
     {
         ActivityHostProcess process = CreateNewActivityHostProcess();
         _hostProcesses.Add(process);
     }
 }
예제 #5
0
        private ActivityHostProcess CreateNewActivityHostProcess()
        {
            ActivityHostProcess activityHostProcess = new ActivityHostProcess(this._configuration.ActivityProcessIdleTimeoutSec, this._configuration.LanguageMode);

            activityHostProcess.ProcessCrashed += new EventHandler <ActivityHostCrashedEventArgs>(this.ProcessCrashed);
            activityHostProcess.Finished       += new EventHandler(this.ProcessFinished);
            activityHostProcess.OnProcessIdle  += new EventHandler(this.ProcessIdle);
            PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)1, true);
            return(activityHostProcess);
        }
        private ActivityHostProcess CreateNewActivityHostProcess()
        {
            ActivityHostProcess process = new ActivityHostProcess(_configuration.ActivityProcessIdleTimeoutSec, _configuration.LanguageMode);

            process.ProcessCrashed += ProcessCrashed;
            process.Finished       += ProcessFinished;
            process.OnProcessIdle  += ProcessIdle;

            PerfCountersMgr.UpdateCounterByValue(
                PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize);
            return(process);
        }
예제 #7
0
        private void ProcessCrashed(object sender, ActivityHostCrashedEventArgs e)
        {
            ActivityHostProcess activityHostProcess = sender as ActivityHostProcess;

            activityHostProcess.MarkForRemoval = true;
            if (e.FailureOnSetup)
            {
                this._failedRequests.Enqueue(e.Invoker);
                PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 17, (long)1, true);
                PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 18, (long)1, true);
            }
            this.DecrementHostCountAndStartThreads();
        }
        /// <summary>
        /// Unregisters all wait handles and disposes a process
        /// </summary>
        /// <param name="process"></param>
        private void SafelyDisposeProcess(ActivityHostProcess process)
        {
            process.Finished       -= ProcessFinished;
            process.ProcessCrashed -= ProcessCrashed;
            process.OnProcessIdle  -= ProcessIdle;

            process.Dispose();

            _hostProcesses.Remove(process);

            PerfCountersMgr.UpdateCounterByValue(
                PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize,
                -1);
        }
예제 #9
0
 private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
 {
     if (!invoker.IsCancelled)
     {
         process.Busy = true;
         Interlocked.Increment(ref this._busyHosts);
         PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 16, (long)1, true);
         Tuple <ActivityHostProcess, ActivityInvoker> tuple = new Tuple <ActivityHostProcess, ActivityInvoker>(process, invoker);
         ThreadPool.QueueUserWorkItem(new WaitCallback(PSOutOfProcessActivityController.RunPowerShellInActivityHostWorker), tuple);
         return;
     }
     else
     {
         return;
     }
 }
        /// <summary>
        /// Method called by servicing thread. This method will run the command in the
        /// specified process on a separate thread
        /// </summary>
        /// <param name="invoker"></param>
        /// <param name="process"></param>
        private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
        {
            if (invoker.IsCancelled)
            {
                return;
            }

            process.Busy = true;
            Interlocked.Increment(ref _busyHosts);
            PerfCountersMgr.UpdateCounterByValue(
                PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                PSWorkflowPerformanceCounterIds.ActivityHostMgrBusyProcessesCount);

            // Adding Tuple object with HostProcess and ActivityInvoker to ThreadPool.QueueUserWorkItem
            // Back reference on ActivityHostProcess in ActivityInvoker is removed
            //
            Tuple <ActivityHostProcess, ActivityInvoker> tupleProcessAndInvoker = new Tuple <ActivityHostProcess, ActivityInvoker>(process, invoker);

            ThreadPool.QueueUserWorkItem(RunPowerShellInActivityHostWorker, tupleProcessAndInvoker);
        }
예제 #11
0
        /// <summary>
        /// Method called by servicing thread. This method will run the command in the 
        /// specified process on a separate thread
        /// </summary>
        /// <param name="invoker"></param>
        /// <param name="process"></param>
        private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
        {
            if (invoker.IsCancelled)
            {
                return;
            }

            process.Busy = true;
            Interlocked.Increment(ref _busyHosts);
            PerfCountersMgr.UpdateCounterByValue(
                    PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                    PSWorkflowPerformanceCounterIds.ActivityHostMgrBusyProcessesCount);

            // Adding Tuple object with HostProcess and ActivityInvoker to ThreadPool.QueueUserWorkItem
            // Back reference on ActivityHostProcess in ActivityInvoker is removed
            //
            Tuple<ActivityHostProcess, ActivityInvoker> tupleProcessAndInvoker = new Tuple<ActivityHostProcess, ActivityInvoker>(process, invoker);
            ThreadPool.QueueUserWorkItem(RunPowerShellInActivityHostWorker, tupleProcessAndInvoker);
        }
예제 #12
0
        /// <summary>
        /// Unregisters all wait handles and disposes a process
        /// </summary>
        /// <param name="process"></param>
        private void SafelyDisposeProcess(ActivityHostProcess process)
        {
            process.Finished -= ProcessFinished;
            process.ProcessCrashed -= ProcessCrashed;
            process.OnProcessIdle -= ProcessIdle;

            process.Dispose();

            _hostProcesses.Remove(process);

            PerfCountersMgr.UpdateCounterByValue(
                    PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                    PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize,
                    -1);
        }
예제 #13
0
        private ActivityHostProcess CreateNewActivityHostProcess()
        {
            ActivityHostProcess process = new ActivityHostProcess(_configuration.ActivityProcessIdleTimeoutSec, _configuration.LanguageMode);
            process.ProcessCrashed += ProcessCrashed;
            process.Finished += ProcessFinished;
            process.OnProcessIdle += ProcessIdle;

            PerfCountersMgr.UpdateCounterByValue(
                 PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                 PSWorkflowPerformanceCounterIds.ActivityHostMgrProcessesPoolSize);
            return process;
        }
        /// <summary>
        /// Method which performs the actual servicing of requests
        /// </summary>
        /// <param name="state"></param>
        private void ServiceRequests(object state)
        {
            bool isFailedRequest = false;

            while (Interlocked.CompareExchange(ref _busyHosts, _configuration.MaxActivityProcesses, _configuration.MaxActivityProcesses) < _configuration.MaxActivityProcesses)
            {
                // if there are any processes marked for removal
                // remove them
                List <ActivityHostProcess> toRemove = _hostProcesses.Where(process => process.MarkForRemoval).ToList();

                foreach (var process in toRemove)
                {
                    SafelyDisposeProcess(process);
                }

                ActivityInvoker invoker;
                // first service previously failed request
                // and then a queued request
                if (_failedRequests.Count > 0)
                {
                    _failedRequests.TryDequeue(out invoker);
                    isFailedRequest = true;
                }
                else
                {
                    _requests.TryDequeue(out invoker);
                    isFailedRequest = false;
                }

                if (invoker == null)
                {
                    break;
                }

                if (invoker.IsCancelled)
                {
                    continue;
                }

                if (isFailedRequest)
                {
                    PerfCountersMgr.UpdateCounterByValue(
                        PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                        PSWorkflowPerformanceCounterIds.ActivityHostMgrFailedRequestsQueueLength,
                        -1);
                }
                else
                {
                    PerfCountersMgr.UpdateCounterByValue(
                        PSWorkflowPerformanceCounterSetInfo.CounterSetId,
                        PSWorkflowPerformanceCounterIds.ActivityHostMgrPendingRequestsQueueLength,
                        -1);
                }

                bool processed = false;
                foreach (ActivityHostProcess process in _hostProcesses.Where(process => !process.Busy))
                {
                    // we have found the first free process available use it
                    processed = true;
                    RunInProcess(invoker, process);
                    break;
                }

                // if there weren't enough processes, then we create one more
                if (processed)
                {
                    continue;
                }

                ActivityHostProcess hostProcess = CreateNewActivityHostProcess();
                _hostProcesses.Add(hostProcess);
                RunInProcess(invoker, hostProcess);
            }

            // we are all done, set servicing to false
            Interlocked.CompareExchange(ref _isServicing, NotServicing, Servicing);

            // Try to start the servicing thread again if there are any pending requests and activity host processes are available.
            // This is required to fix the race condition between CheckAndStartServicingThread() and ServiceRequests() methods.
            //
            if ((_failedRequests.Count > 0 || _requests.Count > 0) &&
                Interlocked.CompareExchange(ref _busyHosts, _configuration.MaxActivityProcesses, _configuration.MaxActivityProcesses) < _configuration.MaxActivityProcesses)
            {
                CheckAndStartServicingThread();
            }
        }
		private void SafelyDisposeProcess(ActivityHostProcess process)
		{
			process.Finished -= new EventHandler(this.ProcessFinished);
			process.ProcessCrashed -= new EventHandler<ActivityHostCrashedEventArgs>(this.ProcessCrashed);
			process.OnProcessIdle -= new EventHandler(this.ProcessIdle);
			process.Dispose();
			this._hostProcesses.Remove(process);
			PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)-1, true);
		}
		private void RunInProcess(ActivityInvoker invoker, ActivityHostProcess process)
		{
			if (!invoker.IsCancelled)
			{
				process.Busy = true;
				Interlocked.Increment(ref this._busyHosts);
				PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 16, (long)1, true);
				Tuple<ActivityHostProcess, ActivityInvoker> tuple = new Tuple<ActivityHostProcess, ActivityInvoker>(process, invoker);
				ThreadPool.QueueUserWorkItem(new WaitCallback(PSOutOfProcessActivityController.RunPowerShellInActivityHostWorker), tuple);
				return;
			}
			else
			{
				return;
			}
		}
		private ActivityHostProcess CreateNewActivityHostProcess()
		{
			ActivityHostProcess activityHostProcess = new ActivityHostProcess(this._configuration.ActivityProcessIdleTimeoutSec, this._configuration.LanguageMode);
			activityHostProcess.ProcessCrashed += new EventHandler<ActivityHostCrashedEventArgs>(this.ProcessCrashed);
			activityHostProcess.Finished += new EventHandler(this.ProcessFinished);
			activityHostProcess.OnProcessIdle += new EventHandler(this.ProcessIdle);
			PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 23, (long)1, true);
			return activityHostProcess;
		}
예제 #18
0
        private void ServiceRequests(object state)
        {
            ActivityInvoker activityInvoker = null;
            bool            flag            = false;

            while (Interlocked.CompareExchange(ref this._busyHosts, this._configuration.MaxActivityProcesses, this._configuration.MaxActivityProcesses) < this._configuration.MaxActivityProcesses)
            {
                Collection <ActivityHostProcess> activityHostProcesses = this._hostProcesses;
                List <ActivityHostProcess>       list = activityHostProcesses.Where <ActivityHostProcess>((ActivityHostProcess process) => process.MarkForRemoval).ToList <ActivityHostProcess>();
                foreach (ActivityHostProcess activityHostProcess in list)
                {
                    this.SafelyDisposeProcess(activityHostProcess);
                }
                if (this._failedRequests.Count <= 0)
                {
                    this._requests.TryDequeue(out activityInvoker);
                    flag = false;
                }
                else
                {
                    this._failedRequests.TryDequeue(out activityInvoker);
                    flag = true;
                }
                if (activityInvoker == null)
                {
                    break;
                }
                if (activityInvoker.IsCancelled)
                {
                    continue;
                }
                if (!flag)
                {
                    PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 20, (long)-1, true);
                }
                else
                {
                    PSOutOfProcessActivityController.PerfCountersMgr.UpdateCounterByValue(PSWorkflowPerformanceCounterSetInfo.CounterSetId, 18, (long)-1, true);
                }
                bool flag1 = false;
                Collection <ActivityHostProcess>  activityHostProcesses1 = this._hostProcesses;
                IEnumerator <ActivityHostProcess> enumerator             = activityHostProcesses1.Where <ActivityHostProcess>((ActivityHostProcess process) => !process.Busy).GetEnumerator();
                using (enumerator)
                {
                    if (enumerator.MoveNext())
                    {
                        ActivityHostProcess current = enumerator.Current;
                        flag1 = true;
                        this.RunInProcess(activityInvoker, current);
                    }
                }
                if (flag1)
                {
                    continue;
                }
                ActivityHostProcess activityHostProcess1 = this.CreateNewActivityHostProcess();
                this._hostProcesses.Add(activityHostProcess1);
                this.RunInProcess(activityInvoker, activityHostProcess1);
            }
            Interlocked.CompareExchange(ref this._isServicing, 0, 1);
            if ((this._failedRequests.Count > 0 || this._requests.Count > 0) && Interlocked.CompareExchange(ref this._busyHosts, this._configuration.MaxActivityProcesses, this._configuration.MaxActivityProcesses) < this._configuration.MaxActivityProcesses)
            {
                this.CheckAndStartServicingThread();
            }
        }