void INodeProvider.ShutdownNodes(Node.NodeShutdownLevel nodeShutdownLevel) { for (int i = 0; i < NodeDescriptions.Count; i++) { NodeDescriptions[i] = null; } }
/// <summary> /// This function is used to decrement the count of active nodes /// </summary> internal void RecordNodeResponse(int nodeId, Node.NodeShutdownLevel shutdownLevel, int totalTaskTime) { // If the node is shutting down - decrease the count of active nodes if (shutdownLevel == Node.NodeShutdownLevel.ErrorShutdown || shutdownLevel == Node.NodeShutdownLevel.PoliteShutdown) { DecreaseActiveNodeCount(nodeId); } //Console.WriteLine("Node " + nodeId + " Task Time " + totalTaskTime); int i = 0; for (; i < nodeData.Length; i++) { if (nodeData[i].NodeId == nodeId) { nodeData[i].ShutdownResponseReceived = true; Interlocked.Decrement(ref responseCount); responseCountChangeEvent.Set(); break; } } ErrorUtilities.VerifyThrow(i < nodeData.Length, "Expected to find a node to decrement count"); }
/// <summary> /// Shutdown the nodes which are being tracked and managed by this localNodeProvider. /// </summary> public void ShutdownNodes(Node.NodeShutdownLevel nodeShutdownLevel) { // Indicate that nodes should no longer be launched shuttingDown = true; // Send out shutdown requests to all active launched nodes responseCount = activeNodeCount; SendShutdownRequests(nodeShutdownLevel); DateTime startTime = DateTime.Now; // Wait for all nodes to shutdown bool timeoutExpired = false; // Loop until we are ready to shutdown. We are ready to shutdown when // all nodes either have sent their shutdown completed response or they are dead. // Secondly, we will exit the loop if our shudtownTimeout has expired TimeSpan shutdownTimeoutSpan = new TimeSpan(0, 0, shutdownTimeout); while (!ReadyToShutdown() && !timeoutExpired) { responseCountChangeEvent.WaitOne(shutdownResponseTimeout, false); responseCountChangeEvent.Reset(); // Timeout when the loop has been executing for more than shutdownTimeout seconds. timeoutExpired = DateTime.Now.Subtract(startTime) >= shutdownTimeoutSpan; } if (timeoutExpired) { foreach (LocalNodeInfo nodeInfo in nodeData) { //Terminate all of the nodes which have valid processId's but for which we // have not recieved a shutdown response if ((nodeInfo.ProcessId > 0 && !nodeInfo.ShutdownResponseReceived)) { TerminateChildNode(nodeInfo.ProcessId); } } } // Reset the shutdown response received properties incase the nodes are going // to be used for another build on the same engine. foreach (LocalNodeInfo nodeInfo in nodeData) { nodeInfo.ShutdownResponseReceived = false; } // If all nodes are exiting - exit the communication threads if (nodeShutdownLevel != Node.NodeShutdownLevel.BuildCompleteSuccess && nodeShutdownLevel != Node.NodeShutdownLevel.BuildCompleteFailure) { exitCommunicationThreads.Set(); } shuttingDown = false; }
/// <summary> /// This method will shutdown the node being hosted by the child process and notify the parent process if requested, /// </summary> /// <param name="shutdownLevel">What kind of shutdown is causing the child node to shutdown</param> /// <param name="exitProcess">should the child process exit as part of the shutdown process</param> /// <param name="noParentNotification">Indicates if the parent process should be notified the child node is being shutdown</param> internal void ShutdownNode(Node.NodeShutdownLevel shutdownLevel, bool exitProcess, bool noParentNotification) { if (node != null) { try { node.ShutdownNode(shutdownLevel); if (!noParentNotification) { // Write the last event out directly LocalCallDescriptorForShutdownComplete callDescriptor = new LocalCallDescriptorForShutdownComplete(shutdownLevel, node.TotalTaskTime); // Post the message indicating that the shutdown is complete engineCallback.PostMessageToParent(callDescriptor, true); } } catch (Exception e) { if (shutdownLevel != Node.NodeShutdownLevel.ErrorShutdown) { ReportNonFatalCommunicationError(e); } } } // If the shutdownLevel is not a build complete message, then this means there was a politeshutdown or an error shutdown, null the node out // as either it is no longer needed due to the node goign idle or there was a error and it is now in a bad state. if (shutdownLevel != Node.NodeShutdownLevel.BuildCompleteSuccess && shutdownLevel != Node.NodeShutdownLevel.BuildCompleteFailure) { node = null; notInUseEvent.Set(); } if (exitProcess) { // Even if we completed a build, if we are goign to exit the process we need to null out the node and set the notInUseEvent, this is // accomplished by calling this method again with the ErrorShutdown handle if (shutdownLevel == Node.NodeShutdownLevel.BuildCompleteSuccess || shutdownLevel == Node.NodeShutdownLevel.BuildCompleteFailure) { ShutdownNode(Node.NodeShutdownLevel.ErrorShutdown, false, true); } // Signal all the communication threads to exit shutdownEvent.Set(); } }
/// <summary> /// Send shutdown message to the launched child nodes. /// </summary> private void SendShutdownRequests(Node.NodeShutdownLevel nodeShutdownLevel) { for (int i = 0; i < nodeData.Length; i++) { // If there is a node launch in progress wait for it complete or fail // before shutting down the node while (nodeData[i].NodeState == NodeState.LaunchInProgress && !nodeData[i].CommunicationFailed) { Thread.Sleep(500); } if (nodeData[i].NodeState == NodeState.Launched) { if (!nodeData[i].CommunicationFailed) { bool exitProcess = !enableNodeReuse; // If we are shutting down due to a BuildComplete then dont kill the nodes as this method will be called again in the engine shutdown method if (nodeShutdownLevel == Node.NodeShutdownLevel.BuildCompleteSuccess || nodeShutdownLevel == Node.NodeShutdownLevel.BuildCompleteFailure) { exitProcess = false; } // Signal to the node to shutdown LocalCallDescriptorForShutdownNode callDescriptor = new LocalCallDescriptorForShutdownNode(nodeShutdownLevel, exitProcess); nodeData[i].NodeCommandQueue.Enqueue(callDescriptor); } else { TerminateChildNode(nodeData[i].ProcessId); } if (nodeShutdownLevel != Node.NodeShutdownLevel.BuildCompleteSuccess && nodeShutdownLevel != Node.NodeShutdownLevel.BuildCompleteFailure) { nodeData[i].NodeState = NodeState.NotLaunched; } } } }
/// <summary> /// Shut down each of the nodes for all providers registered to the node manager. /// Shuts down the TEM. /// </summary> internal void ShutdownNodes(Node.NodeShutdownLevel nodeShutdownLevel) { foreach (INodeProvider nodeProvider in nodeProviders) { nodeProvider.ShutdownNodes(nodeShutdownLevel); } // Don't shutdown the TEM if the engine maybe reused for another build if (nodeShutdownLevel != Node.NodeShutdownLevel.BuildCompleteFailure && nodeShutdownLevel != Node.NodeShutdownLevel.BuildCompleteSuccess) { if (taskExecutionModule != null) { taskExecutionModule.Shutdown(); taskExecutionModule = null; // At this point we have nulled out the task execution module and have told our task worker threads to exit // we do not want the engine build loop to continue to do any work becasue the operations of the build loop // require the task execution module in many cases. Before this fix, when the engine build loop was allowed // to do work after the task execution module we would get random null reference excetpions depending on // what was the first line to use the TEM after it was nulled out. parentEngine.SetEngineAbortTo(true); } } }
internal override void CreateFromStream(BinaryReader reader) { base.CreateFromStream(reader); shutdownLevel = (Node.NodeShutdownLevel)reader.ReadInt32(); totalTaskTime = reader.ReadInt32(); }
internal LocalCallDescriptorForShutdownComplete(Node.NodeShutdownLevel shutdownLevel, int totalTaskTime) : base(LocalCallType.ShutdownComplete) { this.shutdownLevel = shutdownLevel; this.totalTaskTime = totalTaskTime; }
internal override void CreateFromStream(BinaryReader reader) { base.CreateFromStream(reader); exitProcess = reader.ReadBoolean(); shutdownLevel = (Node.NodeShutdownLevel)reader.ReadInt32(); }
internal LocalCallDescriptorForShutdownNode(Node.NodeShutdownLevel shutdownLevel, bool exitProcess) : base(LocalCallType.ShutdownNode) { this.exitProcess = exitProcess; this.shutdownLevel = shutdownLevel; }