/// <summary> /// This function is called when the task executes a callback via IBuildEngine interface. A thread /// that currently owns the workitem queue will continue to own it, unless a work item comes in while /// it is inside the callback. A thread that enters the callback no longer owns the current directory and /// environment block, but it will always regain them before returning to the task. /// </summary> /// <param name="nodeProxyId"></param> /// <param name="requestResults"></param> internal static void WaitForResults(int nodeProxyId, BuildResult[] requestResults) { TaskWorkerThread workerThread = TaskWorkerThread.GetWorkerThreadForProxyId(nodeProxyId); workerThread.NodeActionLoop(workerThread.threadActive ? NodeLoopExecutionMode.WaitingActiveThread : NodeLoopExecutionMode.WaitingPassiveThread, nodeProxyId, requestResults); }
internal static void PostBuildResult(BuildResult buildResult) { TaskWorkerThread workerThread = TaskWorkerThread.GetWorkerThreadForProxyId(buildResult.NodeProxyId); lock (workerThread.targetEvaluationResults) { workerThread.targetEvaluationResults.AddLast(new LinkedListNode <BuildResult>(buildResult)); workerThread.LocalDoneNoticeEvent.Set(); } }
internal void PostBuildResult(BuildResult buildResult) { TaskWorkerThread workerThread = GetWorkerThreadForHandleId(buildResult.HandleId); if (workerThread != null) { lock (workerThread.postedBuildResults) { workerThread.postedBuildResults.AddLast(new LinkedListNode <BuildResult>(buildResult)); workerThread.LocalDoneNoticeEvent.Set(); } } }
/// <summary> /// The TaskExecutionModule is a the external view into a subsystem responsible for executing user /// tasks. The subsystem consists of TaskWorkerThread, TaskEngine, TaskExecutionState and EngineProxy. /// The engine thread passes the TaskExecutionState to the TEM which after the task finishes passes the /// results back via the engineCallback. /// </summary> internal TaskExecutionModule ( EngineCallback engineCallback, TaskExecutionModuleMode moduleMode, bool profileExecution ) { this.engineCallback = engineCallback; this.moduleMode = moduleMode; // By default start in breadthFirst traversal. This is done to gather enough work at the start of the build to get all the nodes at least working on something. this.breadthFirstTraversal = true; this.profileExecution = profileExecution; this.totalTaskTime = 0; // Get the node the TEM is running on, this is so the parent engine knows which node is requesting a traversal strategy change. nodeId = engineCallback.GetParentEngine().NodeId; SetBatchRequestSize(); // In singleproc mode the task execution module executes tasks on the engine thread. In multi proc mode a new thread is // created so the TEM can submit tasks to a worker queue which will run the tasks on a new thread. if (moduleMode != TaskExecutionModuleMode.SingleProcMode) { this.isRunningMultipleNodes = true; this.activeThreadCount = 0; this.overallThreadCount = 0; this.threadActiveCountEvent = new ManualResetEvent(false); this.threadOverallCountEvent = new ManualResetEvent(false); this.lastTaskActivity = 0; // Create a worker thread and make it the active node thread workerThread = new TaskWorkerThread(this, profileExecution); workerThread.ActivateThread(); } else { this.isRunningMultipleNodes = false; } }
/// <summary> /// This function is called when the task executes a callback via IBuildEngine interface. A thread /// that currently owns the workitem queue will continue to own it, unless a work item comes in while /// it is inside the callback. A thread that enters the callback no longer owns the current directory and /// environment block, but it will always regain them before returning to the task. /// </summary> internal void WaitForResults ( int handleId, BuildResult[] buildResults, BuildRequest [] buildRequests ) { TaskWorkerThread workerThread = GetWorkerThreadForHandleId(handleId); ErrorUtilities.VerifyThrow(workerThread != null, "Worker thread should be in the table"); WaitingTaskData taskData = new WaitingTaskData(buildRequests, buildResults); lock (waitingTasks) { waitingTasks.Add(handleId, taskData); } workerThread.NodeActionLoop(workerThread.threadActive ? NodeLoopExecutionMode.WaitingActiveThread : NodeLoopExecutionMode.WaitingPassiveThread, handleId, buildResults); lock (waitingTasks) { waitingTasks.Remove(handleId); } }
/// <summary> /// This is the loop for all active threads. Depending on the current execution mode the thread /// will listen to different events. There is only one thread at the time that owns the workitem /// queue and listens for tasks to be executed. There is also only one thread at a time that is /// execution a task. That thread owns the current directory and the environment block. /// </summary> private void NodeActionLoop ( NodeLoopExecutionMode executionMode, int handleId, BuildResult [] buildResults ) { // Create an array of event to the node thread responds WaitHandle[] waitHandles = GetHandlesArray(executionMode); int resultCount = 0; long entryTime = 0; // A thread that is waiting for a done notification is no longer // actively executing a task so the active cound needs to be decreased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { parentModule.DecrementActiveThreadCount(); // If requested measure the time spent waiting for the results if (profileExecution) { entryTime = DateTime.Now.Ticks; } } bool continueExecution = true; while (continueExecution) { int eventType; // Try and avoid the wait on kernel objects if possible. if (!WaitAnyFast(executionMode, out eventType)) { eventType = WaitHandle.WaitAny(waitHandles); } if (Engine.debugMode) { Console.WriteLine("TaskWorkerThread: HandleId " + handleId + " waking up due to event type " + eventType); } // Node exit event - all threads need to exit if (eventType == 0) { continueExecution = false; } // New work item has appeared in the queue else if (eventType == 1 && executionMode != NodeLoopExecutionMode.WaitingPassiveThread) { ErrorUtilities.VerifyThrow( executionMode == NodeLoopExecutionMode.WaitingActiveThread || executionMode == NodeLoopExecutionMode.BaseActiveThread, "Only active threads should receive work item events"); if (executionMode == NodeLoopExecutionMode.BaseActiveThread) { // Wait until all there are no other active threads, we // always transition from 0 to 1 atomically before executing the task parentModule.WaitForZeroActiveThreadCount(); TaskExecutionState taskToExecute = null; lock (workItemQueue) { taskToExecute = workItemQueue.Dequeue(); // We may get a single event for multiple messages so only reset the event // if the queue is empty if (workItemQueue.Count == 0) { workItemInsertionEvent.Reset(); } } // Execute the task either directly or on a child thread ErrorUtilities.VerifyThrow(taskToExecute != null, "Expected a workitem"); handleIdToWorkerThread[taskToExecute.HandleId] = this; currentWorkitem = taskToExecute; // Actually execute the task (never throws - all exceptions are captured) taskToExecute.ExecuteTask(); currentWorkitem = null; handleIdToWorkerThread.Remove(taskToExecute.HandleId); // Indicate that this thread is no longer active parentModule.DecrementActiveThreadCount(); } else { // Change the thread execution mode since it will no longer be // listening to the work item queue executionMode = NodeLoopExecutionMode.WaitingPassiveThread; threadActive = false; waitHandles = GetHandlesArray(executionMode); TaskWorkerThread workerThread = null; lock (workerThreadQueue) { if (workerThreadQueue.Count != 0) { //Console.WriteLine("REUSING a thread"); workerThread = workerThreadQueue.Dequeue(); } } if (workerThread == null) { //Console.WriteLine("CREATING a thread"); workerThread = new TaskWorkerThread(parentModule, exitTaskThreads, exitTaskThreadsCache, workerThreadQueue, handleIdToWorkerThread, workItemQueue, workItemInsertionEvent, waitingTasks, profileExecution); } workerThread.ActivateThread(); } } else if ((eventType == 1 && executionMode == NodeLoopExecutionMode.WaitingPassiveThread) || (eventType == 2 && executionMode == NodeLoopExecutionMode.WaitingActiveThread)) { // There maybe multiple results in the list so we need to loop over it // and store the results int originalResultCount = resultCount; lock (postedBuildResults) { LinkedListNode <BuildResult> currentNode = postedBuildResults.First; while (currentNode != null) { BuildResult buildResult = currentNode.Value; ErrorUtilities.VerifyThrow( buildResult.RequestId < buildResults.Length, "The request ids should be inside the array"); buildResults[buildResult.RequestId] = buildResult; // Increment the result count to indicate that we got another result resultCount++; // Go to the next entry in the list (most of the time there will be just one entry) currentNode = currentNode.Next; } postedBuildResults.Clear(); // Reset the handle now that we done with the events int handleIndex = executionMode == NodeLoopExecutionMode.WaitingPassiveThread ? 1 : 2; ((ManualResetEvent)waitHandles[handleIndex]).Reset(); } ErrorUtilities.VerifyThrow(originalResultCount < resultCount, "We should have found at least 1 result"); // If we received results for all the requests we need to exit if (resultCount == buildResults.Length) { continueExecution = false; } } // Check if we need to update the state if (executionMode == NodeLoopExecutionMode.BaseActiveThread && !threadActive) { continueExecution = false; } } ErrorUtilities.VerifyThrow (resultCount == 0 || executionMode != NodeLoopExecutionMode.BaseActiveThread, "The base thread should never return a value"); // If a thread exits this loop it is back to actively executing the task, // so the active thread count has to be increased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { parentModule.WaitForZeroActiveThreadCount(); // Sent the time spent waiting for results to the ExecutionState so that the task execution time can be measured correctly if (profileExecution) { this.currentWorkitem.NotifyOfWait(entryTime); } } }
/// <summary> /// This is the loop for all active threads. Depending on the current execution mode the thread /// will listen to different events. There is only one thread at the time that owns the workitem /// queue and listens for tasks to be executed. There is also only one thread at a time that is /// execution a task. That thread owns the current directory and the environment block. /// </summary> /// <param name="executionMode"></param> /// <param name="nodeProxyId"></param> /// <param name="requestResults"></param> private void NodeActionLoop(NodeLoopExecutionMode executionMode, int nodeProxyId, BuildResult [] requestResults) { // Create an array of event to the node thread responds WaitHandle[] waitHandles = GetHandlesArray(executionMode); int resultCount = 0; // A thread that is waiting for a done notification is no longer // actively executing a task so the active cound needs to be decreased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { Interlocked.Decrement(ref activeThreadCount); threadCountEvent.Set(); } bool continueExecution = true; while (continueExecution) { int eventType = WaitHandle.WaitAny(waitHandles); if (Environment.GetEnvironmentVariable("MSBUILDDEBUG") == "1") { Console.WriteLine("NodeProxy :" + nodeProxyId + " waking up due " + eventType); } // Node exit event - all threads need to exit if (eventType == 0) { continueExecution = false; } // New work item has appeared in the queue else if (eventType == 1 && executionMode != NodeLoopExecutionMode.WaitingPassiveThread ) { ErrorUtilities.VerifyThrow( executionMode == NodeLoopExecutionMode.WaitingActiveThread || executionMode == NodeLoopExecutionMode.BaseActiveThread, "Only active threads should receive work item events"); if (executionMode == NodeLoopExecutionMode.BaseActiveThread) { TaskExecutionState taskToExecute = null; lock (workItemQueue) { taskToExecute = workItemQueue.Dequeue(); // We may get a single event for multiple messages so only reset the event // if the queue is empty if (workItemQueue.Count == 0) { workItemInsertionEvent.Reset(); } } // Execute the task either directly or on a child thread ErrorUtilities.VerifyThrow(taskToExecute != null, "Expected a workitem"); // Wait until all there are no other active threads, we // always transition from 0 to 1 atomically before executing the task while (Interlocked.CompareExchange(ref activeThreadCount, 1, 0) != 0) { threadCountEvent.WaitOne(); threadCountEvent.Reset(); } proxyIdToWorkerThread[taskToExecute.NodeProxyId] = this; // Actually execute the task taskToExecute.ExecuteTask(); proxyIdToWorkerThread.Remove(taskToExecute.NodeProxyId); // Indicate that this thread is no longer active Interlocked.Decrement(ref activeThreadCount); threadCountEvent.Set(); } else { // Change the thread execution mode since it will no longer be // listening to the work item queue executionMode = NodeLoopExecutionMode.WaitingPassiveThread; threadActive = false; waitHandles = GetHandlesArray(executionMode); TaskWorkerThread workerThread = null; lock (workerThreadQueue) { if (workerThreadQueue.Count != 0) { //Console.WriteLine("REUSING a thread"); workerThread = workerThreadQueue.Dequeue(); } } if (workerThread == null) { //Console.WriteLine("CREATING a thread"); workerThread = new TaskWorkerThread(); } workerThread.ActivateThread(); } } else if (eventType == 1 && executionMode == NodeLoopExecutionMode.WaitingPassiveThread || eventType == 2 && executionMode == NodeLoopExecutionMode.WaitingActiveThread) { // There maybe multiple results in the list so we need to loop over it // and store the results int originalResultCount = resultCount; lock (targetEvaluationResults) { //Console.WriteLine("Worker thread for: " + nodeProxyId + " Got results"); LinkedListNode<BuildResult> currentNode = targetEvaluationResults.First; while (currentNode != null) { BuildResult buildResult = currentNode.Value; ErrorUtilities.VerifyThrow( buildResult.RequestId < requestResults.Length, "The request ids should be inside the array"); requestResults[buildResult.RequestId] = buildResult; // Increment the result count to indicate that we got another result resultCount++; // Go to the next entry in the list (most of the time there will be just one entry) currentNode = currentNode.Next; } targetEvaluationResults.Clear(); // Reset the handle now that we done with the events int handleIndex = executionMode == NodeLoopExecutionMode.WaitingPassiveThread ? 1 : 2; ((ManualResetEvent)waitHandles[handleIndex]).Reset(); } ErrorUtilities.VerifyThrow(originalResultCount < resultCount, "We should have found at least 1 result"); // If we received results for all the requests we need to exit if (resultCount == requestResults.Length) { continueExecution = false; } } // Check if we need to update the state if (executionMode == NodeLoopExecutionMode.BaseActiveThread && !threadActive) { continueExecution = false; } } ErrorUtilities.VerifyThrow (resultCount == 0 || executionMode != NodeLoopExecutionMode.BaseActiveThread, "The base thread should never return a value"); // If a thread exits this loop it is back to actively executing the task, // so the active thread count has to be increased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { while (Interlocked.CompareExchange(ref activeThreadCount, 1, 0) != 0) { threadCountEvent.WaitOne(); threadCountEvent.Reset(); } } }
/// <summary> /// This is the loop for all active threads. Depending on the current execution mode the thread /// will listen to different events. There is only one thread at the time that owns the workitem /// queue and listens for tasks to be executed. There is also only one thread at a time that is /// execution a task. That thread owns the current directory and the environment block. /// </summary> /// <param name="executionMode"></param> /// <param name="nodeProxyId"></param> /// <param name="requestResults"></param> private void NodeActionLoop(NodeLoopExecutionMode executionMode, int nodeProxyId, BuildResult [] requestResults) { // Create an array of event to the node thread responds WaitHandle[] waitHandles = GetHandlesArray(executionMode); int resultCount = 0; // A thread that is waiting for a done notification is no longer // actively executing a task so the active cound needs to be decreased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { Interlocked.Decrement(ref activeThreadCount); threadCountEvent.Set(); } bool continueExecution = true; while (continueExecution) { int eventType = WaitHandle.WaitAny(waitHandles); if (Environment.GetEnvironmentVariable("MSBUILDDEBUG") == "1") { Console.WriteLine("NodeProxy :" + nodeProxyId + " waking up due " + eventType); } // Node exit event - all threads need to exit if (eventType == 0) { continueExecution = false; } // New work item has appeared in the queue else if (eventType == 1 && executionMode != NodeLoopExecutionMode.WaitingPassiveThread) { ErrorUtilities.VerifyThrow( executionMode == NodeLoopExecutionMode.WaitingActiveThread || executionMode == NodeLoopExecutionMode.BaseActiveThread, "Only active threads should receive work item events"); if (executionMode == NodeLoopExecutionMode.BaseActiveThread) { TaskExecutionState taskToExecute = null; lock (workItemQueue) { taskToExecute = workItemQueue.Dequeue(); // We may get a single event for multiple messages so only reset the event // if the queue is empty if (workItemQueue.Count == 0) { workItemInsertionEvent.Reset(); } } // Execute the task either directly or on a child thread ErrorUtilities.VerifyThrow(taskToExecute != null, "Expected a workitem"); // Wait until all there are no other active threads, we // always transition from 0 to 1 atomically before executing the task while (Interlocked.CompareExchange(ref activeThreadCount, 1, 0) != 0) { threadCountEvent.WaitOne(); threadCountEvent.Reset(); } proxyIdToWorkerThread[taskToExecute.NodeProxyId] = this; // Actually execute the task taskToExecute.ExecuteTask(); proxyIdToWorkerThread.Remove(taskToExecute.NodeProxyId); // Indicate that this thread is no longer active Interlocked.Decrement(ref activeThreadCount); threadCountEvent.Set(); } else { // Change the thread execution mode since it will no longer be // listening to the work item queue executionMode = NodeLoopExecutionMode.WaitingPassiveThread; threadActive = false; waitHandles = GetHandlesArray(executionMode); TaskWorkerThread workerThread = null; lock (workerThreadQueue) { if (workerThreadQueue.Count != 0) { //Console.WriteLine("REUSING a thread"); workerThread = workerThreadQueue.Dequeue(); } } if (workerThread == null) { //Console.WriteLine("CREATING a thread"); workerThread = new TaskWorkerThread(); } workerThread.ActivateThread(); } } else if (eventType == 1 && executionMode == NodeLoopExecutionMode.WaitingPassiveThread || eventType == 2 && executionMode == NodeLoopExecutionMode.WaitingActiveThread) { // There maybe multiple results in the list so we need to loop over it // and store the results int originalResultCount = resultCount; lock (targetEvaluationResults) { //Console.WriteLine("Worker thread for: " + nodeProxyId + " Got results"); LinkedListNode <BuildResult> currentNode = targetEvaluationResults.First; while (currentNode != null) { BuildResult buildResult = currentNode.Value; ErrorUtilities.VerifyThrow( buildResult.RequestId < requestResults.Length, "The request ids should be inside the array"); requestResults[buildResult.RequestId] = buildResult; // Increment the result count to indicate that we got another result resultCount++; // Go to the next entry in the list (most of the time there will be just one entry) currentNode = currentNode.Next; } targetEvaluationResults.Clear(); // Reset the handle now that we done with the events int handleIndex = executionMode == NodeLoopExecutionMode.WaitingPassiveThread ? 1 : 2; ((ManualResetEvent)waitHandles[handleIndex]).Reset(); } ErrorUtilities.VerifyThrow(originalResultCount < resultCount, "We should have found at least 1 result"); // If we received results for all the requests we need to exit if (resultCount == requestResults.Length) { continueExecution = false; } } // Check if we need to update the state if (executionMode == NodeLoopExecutionMode.BaseActiveThread && !threadActive) { continueExecution = false; } } ErrorUtilities.VerifyThrow (resultCount == 0 || executionMode != NodeLoopExecutionMode.BaseActiveThread, "The base thread should never return a value"); // If a thread exits this loop it is back to actively executing the task, // so the active thread count has to be increased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { while (Interlocked.CompareExchange(ref activeThreadCount, 1, 0) != 0) { threadCountEvent.WaitOne(); threadCountEvent.Reset(); } } }
/// <summary> /// This is the loop for all active threads. Depending on the current execution mode the thread /// will listen to different events. There is only one thread at the time that owns the workitem /// queue and listens for tasks to be executed. There is also only one thread at a time that is /// execution a task. That thread owns the current directory and the environment block. /// </summary> private void NodeActionLoop ( NodeLoopExecutionMode executionMode, int handleId, BuildResult [] buildResults ) { // Create an array of event to the node thread responds WaitHandle[] waitHandles = GetHandlesArray(executionMode); int resultCount = 0; long entryTime = 0; // A thread that is waiting for a done notification is no longer // actively executing a task so the active cound needs to be decreased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { parentModule.DecrementActiveThreadCount(); // If requested measure the time spent waiting for the results if (profileExecution) { entryTime = DateTime.Now.Ticks; } } bool continueExecution = true; while (continueExecution) { int eventType; // Try and avoid the wait on kernel objects if possible. if (!WaitAnyFast(executionMode, out eventType)) { eventType = WaitHandle.WaitAny(waitHandles); } if (Engine.debugMode) { Console.WriteLine("TaskWorkerThread: HandleId " + handleId + " waking up due to event type " + eventType); } // Node exit event - all threads need to exit if (eventType == 0) { continueExecution = false; } // New work item has appeared in the queue else if (eventType == 1 && executionMode != NodeLoopExecutionMode.WaitingPassiveThread ) { ErrorUtilities.VerifyThrow( executionMode == NodeLoopExecutionMode.WaitingActiveThread || executionMode == NodeLoopExecutionMode.BaseActiveThread, "Only active threads should receive work item events"); if (executionMode == NodeLoopExecutionMode.BaseActiveThread) { // Wait until all there are no other active threads, we // always transition from 0 to 1 atomically before executing the task parentModule.WaitForZeroActiveThreadCount(); TaskExecutionState taskToExecute = null; lock (workItemQueue) { taskToExecute = workItemQueue.Dequeue(); // We may get a single event for multiple messages so only reset the event // if the queue is empty if (workItemQueue.Count == 0) { workItemInsertionEvent.Reset(); } } // Execute the task either directly or on a child thread ErrorUtilities.VerifyThrow(taskToExecute != null, "Expected a workitem"); handleIdToWorkerThread[taskToExecute.HandleId] = this; currentWorkitem = taskToExecute; // Actually execute the task (never throws - all exceptions are captured) taskToExecute.ExecuteTask(); currentWorkitem = null; handleIdToWorkerThread.Remove(taskToExecute.HandleId); // Indicate that this thread is no longer active parentModule.DecrementActiveThreadCount(); } else { // Change the thread execution mode since it will no longer be // listening to the work item queue executionMode = NodeLoopExecutionMode.WaitingPassiveThread; threadActive = false; waitHandles = GetHandlesArray(executionMode); TaskWorkerThread workerThread = null; lock (workerThreadQueue) { if (workerThreadQueue.Count != 0) { //Console.WriteLine("REUSING a thread"); workerThread = workerThreadQueue.Dequeue(); } } if (workerThread == null) { //Console.WriteLine("CREATING a thread"); workerThread = new TaskWorkerThread(parentModule, exitTaskThreads, exitTaskThreadsCache, workerThreadQueue, handleIdToWorkerThread, workItemQueue, workItemInsertionEvent, waitingTasks, profileExecution); } workerThread.ActivateThread(); } } else if (eventType == 1 && executionMode == NodeLoopExecutionMode.WaitingPassiveThread || eventType == 2 && executionMode == NodeLoopExecutionMode.WaitingActiveThread) { // There maybe multiple results in the list so we need to loop over it // and store the results int originalResultCount = resultCount; lock (postedBuildResults) { LinkedListNode<BuildResult> currentNode = postedBuildResults.First; while (currentNode != null) { BuildResult buildResult = currentNode.Value; ErrorUtilities.VerifyThrow( buildResult.RequestId < buildResults.Length, "The request ids should be inside the array"); buildResults[buildResult.RequestId] = buildResult; // Increment the result count to indicate that we got another result resultCount++; // Go to the next entry in the list (most of the time there will be just one entry) currentNode = currentNode.Next; } postedBuildResults.Clear(); // Reset the handle now that we done with the events int handleIndex = executionMode == NodeLoopExecutionMode.WaitingPassiveThread ? 1 : 2; ((ManualResetEvent)waitHandles[handleIndex]).Reset(); } ErrorUtilities.VerifyThrow(originalResultCount < resultCount, "We should have found at least 1 result"); // If we received results for all the requests we need to exit if (resultCount == buildResults.Length) { continueExecution = false; } } // Check if we need to update the state if (executionMode == NodeLoopExecutionMode.BaseActiveThread && !threadActive) { continueExecution = false; } } ErrorUtilities.VerifyThrow (resultCount == 0 || executionMode != NodeLoopExecutionMode.BaseActiveThread, "The base thread should never return a value"); // If a thread exits this loop it is back to actively executing the task, // so the active thread count has to be increased if (executionMode != NodeLoopExecutionMode.BaseActiveThread) { parentModule.WaitForZeroActiveThreadCount(); // Sent the time spent waiting for results to the ExecutionState so that the task execution time can be measured correctly if (profileExecution) { this.currentWorkitem.NotifyOfWait(entryTime); } } }