/// <summary>
/// This function is called when the task executes a callback via IBuildEngine interface. A thread
/// that currently owns the workitem queue will continue to own it, unless a work item comes in while
/// it is inside the callback. A thread that enters the callback no longer owns the current directory and
/// environment block, but it will always regain them before returning to the task.
/// </summary>
/// <param name="nodeProxyId"></param>
/// <param name="requestResults"></param>
internal static void WaitForResults(int nodeProxyId, BuildResult[] requestResults)
{
TaskWorkerThread workerThread = TaskWorkerThread.GetWorkerThreadForProxyId(nodeProxyId);
workerThread.NodeActionLoop(workerThread.threadActive ? NodeLoopExecutionMode.WaitingActiveThread :
NodeLoopExecutionMode.WaitingPassiveThread,
nodeProxyId, requestResults);
}
internal static void PostBuildResult(BuildResult buildResult)
{
TaskWorkerThread workerThread = TaskWorkerThread.GetWorkerThreadForProxyId(buildResult.NodeProxyId);
lock (workerThread.targetEvaluationResults)
{
workerThread.targetEvaluationResults.AddLast(new LinkedListNode <BuildResult>(buildResult));
workerThread.LocalDoneNoticeEvent.Set();
}
}
internal void PostBuildResult(BuildResult buildResult)
{
TaskWorkerThread workerThread = GetWorkerThreadForHandleId(buildResult.HandleId);
if (workerThread != null)
{
lock (workerThread.postedBuildResults)
{
workerThread.postedBuildResults.AddLast(new LinkedListNode <BuildResult>(buildResult));
workerThread.LocalDoneNoticeEvent.Set();
}
}
}
/// <summary>
/// The TaskExecutionModule is a the external view into a subsystem responsible for executing user
/// tasks. The subsystem consists of TaskWorkerThread, TaskEngine, TaskExecutionState and EngineProxy.
/// The engine thread passes the TaskExecutionState to the TEM which after the task finishes passes the
/// results back via the engineCallback.
/// </summary>
internal TaskExecutionModule
(
EngineCallback engineCallback,
TaskExecutionModuleMode moduleMode,
bool profileExecution
)
{
this.engineCallback = engineCallback;
this.moduleMode = moduleMode;
// By default start in breadthFirst traversal. This is done to gather enough work at the start of the build to get all the nodes at least working on something.
this.breadthFirstTraversal = true;
this.profileExecution = profileExecution;
this.totalTaskTime = 0;
// Get the node the TEM is running on, this is so the parent engine knows which node is requesting a traversal strategy change.
nodeId = engineCallback.GetParentEngine().NodeId;
SetBatchRequestSize();
// In singleproc mode the task execution module executes tasks on the engine thread. In multi proc mode a new thread is
// created so the TEM can submit tasks to a worker queue which will run the tasks on a new thread.
if (moduleMode != TaskExecutionModuleMode.SingleProcMode)
{
this.isRunningMultipleNodes = true;
this.activeThreadCount = 0;
this.overallThreadCount = 0;
this.threadActiveCountEvent = new ManualResetEvent(false);
this.threadOverallCountEvent = new ManualResetEvent(false);
this.lastTaskActivity = 0;
// Create a worker thread and make it the active node thread
workerThread = new TaskWorkerThread(this, profileExecution);
workerThread.ActivateThread();
}
else
{
this.isRunningMultipleNodes = false;
}
}
/// <summary>
/// This function is called when the task executes a callback via IBuildEngine interface. A thread
/// that currently owns the workitem queue will continue to own it, unless a work item comes in while
/// it is inside the callback. A thread that enters the callback no longer owns the current directory and
/// environment block, but it will always regain them before returning to the task.
/// </summary>
internal void WaitForResults
(
int handleId,
BuildResult[] buildResults,
BuildRequest [] buildRequests
)
{
TaskWorkerThread workerThread = GetWorkerThreadForHandleId(handleId);
ErrorUtilities.VerifyThrow(workerThread != null, "Worker thread should be in the table");
WaitingTaskData taskData = new WaitingTaskData(buildRequests, buildResults);
lock (waitingTasks)
{
waitingTasks.Add(handleId, taskData);
}
workerThread.NodeActionLoop(workerThread.threadActive ? NodeLoopExecutionMode.WaitingActiveThread :
NodeLoopExecutionMode.WaitingPassiveThread,
handleId, buildResults);
lock (waitingTasks)
{
waitingTasks.Remove(handleId);
}
}
/// <summary>
/// This is the loop for all active threads. Depending on the current execution mode the thread
/// will listen to different events. There is only one thread at the time that owns the workitem
/// queue and listens for tasks to be executed. There is also only one thread at a time that is
/// execution a task. That thread owns the current directory and the environment block.
/// </summary>
private void NodeActionLoop
(
NodeLoopExecutionMode executionMode,
int handleId,
BuildResult [] buildResults
)
{
// Create an array of event to the node thread responds
WaitHandle[] waitHandles = GetHandlesArray(executionMode);
int resultCount = 0;
long entryTime = 0;
// A thread that is waiting for a done notification is no longer
// actively executing a task so the active cound needs to be decreased
if (executionMode != NodeLoopExecutionMode.BaseActiveThread)
{
parentModule.DecrementActiveThreadCount();
// If requested measure the time spent waiting for the results
if (profileExecution)
{
entryTime = DateTime.Now.Ticks;
}
}
bool continueExecution = true;
while (continueExecution)
{
int eventType;
// Try and avoid the wait on kernel objects if possible.
if (!WaitAnyFast(executionMode, out eventType))
{
eventType = WaitHandle.WaitAny(waitHandles);
}
if (Engine.debugMode)
{
Console.WriteLine("TaskWorkerThread: HandleId " + handleId + " waking up due to event type " + eventType);
}
// Node exit event - all threads need to exit
if (eventType == 0)
{
continueExecution = false;
}
// New work item has appeared in the queue
else if (eventType == 1 && executionMode != NodeLoopExecutionMode.WaitingPassiveThread)
{
ErrorUtilities.VerifyThrow(
executionMode == NodeLoopExecutionMode.WaitingActiveThread ||
executionMode == NodeLoopExecutionMode.BaseActiveThread,
"Only active threads should receive work item events");
if (executionMode == NodeLoopExecutionMode.BaseActiveThread)
{
// Wait until all there are no other active threads, we
// always transition from 0 to 1 atomically before executing the task
parentModule.WaitForZeroActiveThreadCount();
TaskExecutionState taskToExecute = null;
lock (workItemQueue)
{
taskToExecute = workItemQueue.Dequeue();
// We may get a single event for multiple messages so only reset the event
// if the queue is empty
if (workItemQueue.Count == 0)
{
workItemInsertionEvent.Reset();
}
}
// Execute the task either directly or on a child thread
ErrorUtilities.VerifyThrow(taskToExecute != null, "Expected a workitem");
handleIdToWorkerThread[taskToExecute.HandleId] = this;
currentWorkitem = taskToExecute;
// Actually execute the task (never throws - all exceptions are captured)
taskToExecute.ExecuteTask();
currentWorkitem = null;
handleIdToWorkerThread.Remove(taskToExecute.HandleId);
// Indicate that this thread is no longer active
parentModule.DecrementActiveThreadCount();
}
else
{
// Change the thread execution mode since it will no longer be
// listening to the work item queue
executionMode = NodeLoopExecutionMode.WaitingPassiveThread;
threadActive = false;
waitHandles = GetHandlesArray(executionMode);
TaskWorkerThread workerThread = null;
lock (workerThreadQueue)
{
if (workerThreadQueue.Count != 0)
{
//Console.WriteLine("REUSING a thread");
workerThread = workerThreadQueue.Dequeue();
}
}
if (workerThread == null)
{
//Console.WriteLine("CREATING a thread");
workerThread = new TaskWorkerThread(parentModule, exitTaskThreads, exitTaskThreadsCache, workerThreadQueue, handleIdToWorkerThread,
workItemQueue, workItemInsertionEvent, waitingTasks, profileExecution);
}
workerThread.ActivateThread();
}
}
else if (eventType == 1 && executionMode == NodeLoopExecutionMode.WaitingPassiveThread ||
eventType == 2 && executionMode == NodeLoopExecutionMode.WaitingActiveThread)
{
// There maybe multiple results in the list so we need to loop over it
// and store the results
int originalResultCount = resultCount;
lock (postedBuildResults)
{
LinkedListNode <BuildResult> currentNode = postedBuildResults.First;
while (currentNode != null)
{
BuildResult buildResult = currentNode.Value;
ErrorUtilities.VerifyThrow(
buildResult.RequestId < buildResults.Length,
"The request ids should be inside the array");
buildResults[buildResult.RequestId] = buildResult;
// Increment the result count to indicate that we got another result
resultCount++;
// Go to the next entry in the list (most of the time there will be just one entry)
currentNode = currentNode.Next;
}
postedBuildResults.Clear();
// Reset the handle now that we done with the events
int handleIndex = executionMode == NodeLoopExecutionMode.WaitingPassiveThread ? 1 : 2;
((ManualResetEvent)waitHandles[handleIndex]).Reset();
}
ErrorUtilities.VerifyThrow(originalResultCount < resultCount, "We should have found at least 1 result");
// If we received results for all the requests we need to exit
if (resultCount == buildResults.Length)
{
continueExecution = false;
}
}
// Check if we need to update the state
if (executionMode == NodeLoopExecutionMode.BaseActiveThread && !threadActive)
{
continueExecution = false;
}
}
ErrorUtilities.VerifyThrow
(resultCount == 0 || executionMode != NodeLoopExecutionMode.BaseActiveThread,
"The base thread should never return a value");
// If a thread exits this loop it is back to actively executing the task,
// so the active thread count has to be increased
if (executionMode != NodeLoopExecutionMode.BaseActiveThread)
{
parentModule.WaitForZeroActiveThreadCount();
// Sent the time spent waiting for results to the ExecutionState so that the task execution time can be measured correctly
if (profileExecution)
{
this.currentWorkitem.NotifyOfWait(entryTime);
}
}
}
/// <summary>
/// This is the loop for all active threads. Depending on the current execution mode the thread
/// will listen to different events. There is only one thread at the time that owns the workitem
/// queue and listens for tasks to be executed. There is also only one thread at a time that is
/// execution a task. That thread owns the current directory and the environment block.
/// </summary>
/// <param name="executionMode"></param>
/// <param name="nodeProxyId"></param>
/// <param name="requestResults"></param>
private void NodeActionLoop(NodeLoopExecutionMode executionMode, int nodeProxyId, BuildResult [] requestResults)
{
// Create an array of event to the node thread responds
WaitHandle[] waitHandles = GetHandlesArray(executionMode);
int resultCount = 0;
// A thread that is waiting for a done notification is no longer
// actively executing a task so the active cound needs to be decreased
if (executionMode != NodeLoopExecutionMode.BaseActiveThread)
{
Interlocked.Decrement(ref activeThreadCount);
threadCountEvent.Set();
}
bool continueExecution = true;
while (continueExecution)
{
int eventType = WaitHandle.WaitAny(waitHandles);
if (Environment.GetEnvironmentVariable("MSBUILDDEBUG") == "1")
{
Console.WriteLine("NodeProxy :" + nodeProxyId + " waking up due " + eventType);
}
// Node exit event - all threads need to exit
if (eventType == 0)
{
continueExecution = false;
}
// New work item has appeared in the queue
else if (eventType == 1 && executionMode != NodeLoopExecutionMode.WaitingPassiveThread)
{
ErrorUtilities.VerifyThrow(
executionMode == NodeLoopExecutionMode.WaitingActiveThread ||
executionMode == NodeLoopExecutionMode.BaseActiveThread,
"Only active threads should receive work item events");
if (executionMode == NodeLoopExecutionMode.BaseActiveThread)
{
TaskExecutionState taskToExecute = null;
lock (workItemQueue)
{
taskToExecute = workItemQueue.Dequeue();
// We may get a single event for multiple messages so only reset the event
// if the queue is empty
if (workItemQueue.Count == 0)
{
workItemInsertionEvent.Reset();
}
}
// Execute the task either directly or on a child thread
ErrorUtilities.VerifyThrow(taskToExecute != null, "Expected a workitem");
// Wait until all there are no other active threads, we
// always transition from 0 to 1 atomically before executing the task
while (Interlocked.CompareExchange(ref activeThreadCount, 1, 0) != 0)
{
threadCountEvent.WaitOne();
threadCountEvent.Reset();
}
proxyIdToWorkerThread[taskToExecute.NodeProxyId] = this;
// Actually execute the task
taskToExecute.ExecuteTask();
proxyIdToWorkerThread.Remove(taskToExecute.NodeProxyId);
// Indicate that this thread is no longer active
Interlocked.Decrement(ref activeThreadCount);
threadCountEvent.Set();
}
else
{
// Change the thread execution mode since it will no longer be
// listening to the work item queue
executionMode = NodeLoopExecutionMode.WaitingPassiveThread;
threadActive = false;
waitHandles = GetHandlesArray(executionMode);
TaskWorkerThread workerThread = null;
lock (workerThreadQueue)
{
if (workerThreadQueue.Count != 0)
{
//Console.WriteLine("REUSING a thread");
workerThread = workerThreadQueue.Dequeue();
}
}
if (workerThread == null)
{
//Console.WriteLine("CREATING a thread");
workerThread = new TaskWorkerThread();
}
workerThread.ActivateThread();
}
}
else if (eventType == 1 && executionMode == NodeLoopExecutionMode.WaitingPassiveThread ||
eventType == 2 && executionMode == NodeLoopExecutionMode.WaitingActiveThread)
{
// There maybe multiple results in the list so we need to loop over it
// and store the results
int originalResultCount = resultCount;
lock (targetEvaluationResults)
{
//Console.WriteLine("Worker thread for: " + nodeProxyId + " Got results");
LinkedListNode <BuildResult> currentNode = targetEvaluationResults.First;
while (currentNode != null)
{
BuildResult buildResult = currentNode.Value;
ErrorUtilities.VerifyThrow(
buildResult.RequestId < requestResults.Length,
"The request ids should be inside the array");
requestResults[buildResult.RequestId] = buildResult;
// Increment the result count to indicate that we got another result
resultCount++;
// Go to the next entry in the list (most of the time there will be just one entry)
currentNode = currentNode.Next;
}
targetEvaluationResults.Clear();
// Reset the handle now that we done with the events
int handleIndex = executionMode == NodeLoopExecutionMode.WaitingPassiveThread ? 1 : 2;
((ManualResetEvent)waitHandles[handleIndex]).Reset();
}
ErrorUtilities.VerifyThrow(originalResultCount < resultCount, "We should have found at least 1 result");
// If we received results for all the requests we need to exit
if (resultCount == requestResults.Length)
{
continueExecution = false;
}
}
// Check if we need to update the state
if (executionMode == NodeLoopExecutionMode.BaseActiveThread && !threadActive)
{
continueExecution = false;
}
}
ErrorUtilities.VerifyThrow
(resultCount == 0 || executionMode != NodeLoopExecutionMode.BaseActiveThread,
"The base thread should never return a value");
// If a thread exits this loop it is back to actively executing the task,
// so the active thread count has to be increased
if (executionMode != NodeLoopExecutionMode.BaseActiveThread)
{
while (Interlocked.CompareExchange(ref activeThreadCount, 1, 0) != 0)
{
threadCountEvent.WaitOne();
threadCountEvent.Reset();
}
}
}