/// <summary> /// Constructor. /// </summary> public SchedulingEvent(DateTime eventTime, SchedulableRequest request, SchedulableRequestState oldState, SchedulableRequestState newState) { _eventTime = eventTime; _request = request; _oldState = oldState; _newState = newState; }
/// <summary> /// Retrieves a request which has been assigned to a node and is in the executing, blocked or ready states. /// </summary> public SchedulableRequest GetScheduledRequest(int globalRequestId) { SchedulableRequest returnValue = InternalGetScheduledRequestByGlobalRequestId(globalRequestId); ErrorUtilities.VerifyThrow(returnValue != null, "Global Request Id {0} has not been assigned and cannot be retrieved.", globalRequestId); return(returnValue); }
/// <summary> /// Marks this request as being blocked by the specified request. Establishes the correct relationships between the requests. /// </summary> /// <param name="blockingRequest">The request which is blocking this one.</param> /// <param name="activeTargets">The list of targets this request was currently building at the time it became blocked.</param> public void BlockByRequest(SchedulableRequest blockingRequest, string[] activeTargets) { VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Blocked, SchedulableRequestState.Executing }); ErrorUtilities.VerifyThrowArgumentNull(blockingRequest, "blockingRequest"); ErrorUtilities.VerifyThrowArgumentNull(activeTargets, "activeTargets"); // Note that the blocking request will typically be our parent UNLESS it is a request we blocked on because it was executing a target we wanted to execute. // Thus, we do not assert the parent-child relationship here. BlockingRequestKey key = new BlockingRequestKey(blockingRequest.BuildRequest); ErrorUtilities.VerifyThrow(!_requestsWeAreBlockedBy.ContainsKey(key), "We are already blocked by this request."); ErrorUtilities.VerifyThrow(!blockingRequest._requestsWeAreBlocking.Contains(this), "The blocking request thinks it is already blocking us."); // This method is only called when a request reports that it is blocked on other requests. If the request is being blocked by a brand new // request, that request will be unscheduled. If this request is blocked by an in-progress request which was executing a target it needed // to also execute, then that request is not unscheduled (because it was running on the node) and it is not executing (because this condition // can only occur against requests which are executing on the same node and since the request which called this method is the one currently // executing on that node, that means the request it is blocked by must either be itself blocked or ready.) blockingRequest.VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Yielding, SchedulableRequestState.Blocked, SchedulableRequestState.Ready, SchedulableRequestState.Unscheduled }); // Update our list of active targets. This has to be done before we detect circular dependencies because we use this information to detect // re-entrancy circular dependencies. _activeTargetsWhenBlocked = activeTargets; DetectCircularDependency(blockingRequest); _requestsWeAreBlockedBy[key] = blockingRequest; blockingRequest._requestsWeAreBlocking.Add(this); ChangeToState(SchedulableRequestState.Blocked); }
/// <summary> /// Removes associations with all blocking requests and throws an exception. /// </summary> private void CleanupForCircularDependencyAndThrow(SchedulableRequest requestCausingFailure, List <SchedulableRequest> ancestors) { if (_requestsWeAreBlockedBy.Count != 0) { List <SchedulableRequest> tempRequests = new List <SchedulableRequest>(_requestsWeAreBlockedBy.Values); foreach (SchedulableRequest requestWeAreBlockedBy in tempRequests) { BlockingRequestKey key = new BlockingRequestKey(requestWeAreBlockedBy.BuildRequest); DisconnectRequestWeAreBlockedBy(key); } } else { ChangeToState(SchedulableRequestState.Ready); } _activeTargetsWhenBlocked = null; // The blocking request itself is no longer valid if it was unscheduled. if (requestCausingFailure.State == SchedulableRequestState.Unscheduled) { requestCausingFailure.Delete(); } throw new SchedulerCircularDependencyException(requestCausingFailure.BuildRequest, ancestors); }
/// <summary> /// Creates a new request and adds it to the system /// </summary> /// <remarks> /// New requests always go on the front of the queue, because we prefer to build the projects we just received first (depth first, absent /// any particular scheduling algorithm such as in the single-proc case.) /// </remarks> public SchedulableRequest CreateRequest(BuildRequest buildRequest, SchedulableRequest parent) { SchedulableRequest request = new SchedulableRequest(this, buildRequest, parent); request.CreationTime = EventTime; LinkedListNode <SchedulableRequest> requestNode = _unscheduledRequests.AddFirst(request); _unscheduledRequestNodesByRequest[request] = requestNode; // Update the configuration information. HashSet <SchedulableRequest> requests; if (!_configurationToRequests.TryGetValue(request.BuildRequest.ConfigurationId, out requests)) { requests = new HashSet <SchedulableRequest>(); _configurationToRequests[request.BuildRequest.ConfigurationId] = requests; } requests.Add(request); // Update the build hierarchy. if (!_buildHierarchy.ContainsKey(request)) { _buildHierarchy[request] = new List <SchedulableRequest>(8); } if (parent != null) { ErrorUtilities.VerifyThrow(_buildHierarchy.ContainsKey(parent), "Parent doesn't exist in build hierarchy for request {0}", request.BuildRequest.GlobalRequestId); _buildHierarchy[parent].Add(request); } return(request); }
/// <summary> /// Gets the name of the plan file for a specified submission. /// </summary> private string GetPlanName(SchedulableRequest rootRequest) { if (rootRequest == null) { return(null); } return(_configCache[rootRequest.BuildRequest.ConfigurationId].ProjectFullPath + ".buildplan"); }
/// <summary> /// Explicitly grants CPU cores to a request. /// </summary> public void GrantCoresToRequest(int globalRequestId, int coresToGrant) { // Update per-request state. SchedulableRequest request = GetScheduledRequest(globalRequestId); request.GrantedCores += coresToGrant; // Update global state. _grantedCores += coresToGrant; }
/// <summary> /// Explicitly removes previously granted CPU cores from a request. /// </summary> public void RemoveCoresFromRequest(int globalRequestId, int coresToRemove) { // Update per-request state. SchedulableRequest request = GetScheduledRequest(globalRequestId); coresToRemove = Math.Min(request.GrantedCores, coresToRemove); request.GrantedCores -= coresToRemove; // Update global state. _grantedCores -= coresToRemove; }
/// <summary> /// Reads a plan for the specified submission Id. /// </summary> public void ReadPlan(int submissionId, ILoggingService loggingService, BuildEventContext buildEventContext) { if (!BuildParameters.EnableBuildPlan) { return; } SchedulableRequest rootRequest = GetRootRequest(submissionId); if (rootRequest == null) { return; } string planName = GetPlanName(rootRequest); if (String.IsNullOrEmpty(planName)) { return; } if (!File.Exists(planName)) { return; } try { using (StreamReader file = new StreamReader(File.Open(planName, FileMode.Open))) { ReadTimes(file); ReadHierarchy(file); } if (_configIdToData.Count > 0) { AnalyzeData(); } } catch (IOException) { loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("CantReadBuildPlan", planName)); } catch (InvalidDataException) { loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("BuildPlanCorrupt", planName)); } catch (FormatException) { loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("BuildPlanCorrupt", planName)); } }
/// <summary> /// Verifies that the request is scheduled and in the expected state. /// </summary> private void ExpectScheduledRequestState(int globalRequestId, SchedulableRequestState state) { SchedulableRequest request = InternalGetScheduledRequestByGlobalRequestId(globalRequestId); if (request == null) { ErrorUtilities.ThrowInternalError("Request {0} was expected to be in state {1} but is not scheduled at all (it may be unscheduled or may be unknown to the system.)", globalRequestId, state); } else { request.VerifyState(state); } }
/// <summary> /// Detects a circular dependency where the request which is about to block us is already blocked by us, usually as a result /// of it having been previously scheduled in a multiproc scenario, but before this request was able to execute. /// </summary> /// <remarks> /// Let A be 'this' project and B be 'blockingRequest' (the request which is going to block A.) /// An indirect circular dependency exists if there is a dependency path from B to A. If there is no /// existing blocked request B' with the same global request id as B, then there can be no path from B to A because B is a brand new /// request with no other dependencies. If there is an existing blocked request B' with the same global request ID as B, then we /// walk the set of dependencies recursively searching for A. If A is found, we have a circular dependency. /// </remarks> private void DetectIndirectCircularDependency(SchedulableRequest blockingRequest) { // If there is already a blocked request which has the same configuration id as the blocking request and that blocked request is (recursively) // waiting on this request, then that is an indirect circular dependency. SchedulableRequest alternateRequest = _schedulingData.GetBlockedRequestIfAny(blockingRequest.BuildRequest.GlobalRequestId); if (alternateRequest == null) { return; } Stack <SchedulableRequest> requestsToEvaluate = new Stack <SchedulableRequest>(16); HashSet <SchedulableRequest> evaluatedRequests = new HashSet <SchedulableRequest>(); requestsToEvaluate.Push(alternateRequest); while (requestsToEvaluate.Count > 0) { SchedulableRequest requestToEvaluate = requestsToEvaluate.Pop(); // If we make it to a child which is us, then it's a circular dependency. if (requestToEvaluate.BuildRequest.GlobalRequestId == this.BuildRequest.GlobalRequestId) { ThrowIndirectCircularDependency(blockingRequest, requestToEvaluate); } evaluatedRequests.Add(requestToEvaluate); // If the request is not scheduled, it's possible that is because it's been scheduled elsewhere and is blocked. // Follow that path if it exists. if (requestToEvaluate.State == SchedulableRequestState.Unscheduled) { requestToEvaluate = _schedulingData.GetBlockedRequestIfAny(requestToEvaluate.BuildRequest.GlobalRequestId); // If there was no scheduled request to evaluate, move on. if (requestToEvaluate == null || evaluatedRequests.Contains(requestToEvaluate)) { continue; } } // This request didn't cause a circular dependency, check its children. foreach (SchedulableRequest childRequest in requestToEvaluate.RequestsWeAreBlockedBy) { if (!evaluatedRequests.Contains(childRequest)) { requestsToEvaluate.Push(childRequest); } } } }
/// <summary> /// Recursively accumulates the amount of time spent in each configuration. /// </summary> private void RecursiveAccumulateConfigurationTimes(SchedulableRequest request, Dictionary <int, double> accumulatedTimeByConfiguration) { double accumulatedTime; // NOTE: Do we want to count it each time the config appears in the hierarchy? This will inflate the // cost of frequently referenced configurations. accumulatedTimeByConfiguration.TryGetValue(request.BuildRequest.ConfigurationId, out accumulatedTime); accumulatedTimeByConfiguration[request.BuildRequest.ConfigurationId] = accumulatedTime + request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalMilliseconds; foreach (SchedulableRequest childRequest in _schedulingData.GetRequestsByHierarchy(request)) { RecursiveAccumulateConfigurationTimes(childRequest, accumulatedTimeByConfiguration); } }
/// <summary> /// Writes out all of the dependencies for a specified request, recursively. /// </summary> private void RecursiveWriteDependencies(StreamWriter file, SchedulableRequest request) { file.Write(request.BuildRequest.ConfigurationId); foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request)) { file.Write(" {0}", child.BuildRequest.ConfigurationId); } file.WriteLine(); foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request)) { RecursiveWriteDependencies(file, child); } }
/// <summary> /// Build our ancestor list then throw the circular dependency error. /// </summary> private void ThrowIndirectCircularDependency(SchedulableRequest blockingRequest, SchedulableRequest requestToEvaluate) { // We found a request which has the same global request ID as us in a chain which leads from the (already blocked) request // which is trying to block us. Calculate its list of ancestors by walking up the parent list. List <SchedulableRequest> ancestors = new List <SchedulableRequest>(16); while (requestToEvaluate.Parent != null) { ancestors.Add(requestToEvaluate.Parent); requestToEvaluate = requestToEvaluate.Parent; } ancestors.Reverse(); // Because the list should be in the order from root to child. CleanupForCircularDependencyAndThrow(blockingRequest, ancestors); }
/// <summary> /// Writes a plan for the specified submission id. /// </summary> public void WritePlan(int submissionId, ILoggingService loggingService, BuildEventContext buildEventContext) { if (!BuildParameters.EnableBuildPlan) { return; } SchedulableRequest rootRequest = GetRootRequest(submissionId); if (rootRequest == null) { return; } string planName = GetPlanName(rootRequest); if (String.IsNullOrEmpty(planName)) { return; } try { using (StreamWriter file = new StreamWriter(File.Open(planName, FileMode.Create))) { // Write the accumulated configuration times. Dictionary <int, double> accumulatedTimeByConfiguration = new Dictionary <int, double>(); RecursiveAccumulateConfigurationTimes(rootRequest, accumulatedTimeByConfiguration); List <int> configurationsInOrder = new List <int>(accumulatedTimeByConfiguration.Keys); configurationsInOrder.Sort(); foreach (int configId in configurationsInOrder) { file.WriteLine(String.Format(CultureInfo.InvariantCulture, "{0} {1} {2}", configId, accumulatedTimeByConfiguration[configId], _configCache[configId].ProjectFullPath)); } file.WriteLine(); // Write out the dependency information. RecursiveWriteDependencies(file, rootRequest); } } catch (IOException) { loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("CantWriteBuildPlan", planName)); } }
/// <summary> /// Retrieves a set of build requests which have the specified parent. If root is null, this will retrieve all of the /// top-level requests. /// </summary> public IEnumerable <SchedulableRequest> GetRequestsByHierarchy(SchedulableRequest root) { if (root == null) { // Retrieve all requests which are roots of the tree. List <SchedulableRequest> roots = new List <SchedulableRequest>(); foreach (SchedulableRequest key in _buildHierarchy.Keys) { if (key.Parent == null) { roots.Add(key); } } return(roots); } return(_buildHierarchy[root]); }
/// <summary> /// Detects a circular dependency where the blocking request is in our direct ancestor chain. /// </summary> private void DetectDirectCircularDependency(SchedulableRequest blockingRequest) { // A circular dependency occurs when this project (or any of its ancestors) has the same global request id as the // blocking request. List <SchedulableRequest> ancestors = new List <SchedulableRequest>(16); SchedulableRequest currentRequest = this; do { ancestors.Add(currentRequest); if (currentRequest.BuildRequest.GlobalRequestId == blockingRequest.BuildRequest.GlobalRequestId) { // We are directly conflicting with an instance of ourselves. CleanupForCircularDependencyAndThrow(blockingRequest, ancestors); } currentRequest = currentRequest.Parent; }while (currentRequest != null); }
/// <summary> /// Removes the association between this request and the one we are blocked by. /// </summary> private void DisconnectRequestWeAreBlockedBy(BlockingRequestKey blockingRequestKey) { ErrorUtilities.VerifyThrow(_requestsWeAreBlockedBy.ContainsKey(blockingRequestKey), "We are not blocked by the specified request."); SchedulableRequest unblockingRequest = _requestsWeAreBlockedBy[blockingRequestKey]; ErrorUtilities.VerifyThrow(unblockingRequest._requestsWeAreBlocking.Contains(this), "The request unblocking us doesn't think it is blocking us."); _requestsWeAreBlockedBy.Remove(blockingRequestKey); unblockingRequest._requestsWeAreBlocking.Remove(this); // If the request we are blocked by also happens to be unscheduled, remove it as well so we don't try to run it later. This is // because circular dependency errors cause us to fail all outstanding requests on the current request. See BuildRequsetEntry.ReportResult. if (unblockingRequest.State == SchedulableRequestState.Unscheduled) { unblockingRequest.Delete(); } if (_requestsWeAreBlockedBy.Count == 0) { ChangeToState(SchedulableRequestState.Ready); } }
/// <summary> /// Constructor. /// </summary> public SchedulableRequest(SchedulingData collection, BuildRequest request, SchedulableRequest parent) { ErrorUtilities.VerifyThrowArgumentNull(collection, "collection"); ErrorUtilities.VerifyThrowArgumentNull(request, "request"); ErrorUtilities.VerifyThrow((parent == null) || (parent._schedulingData == collection), "Parent request does not belong to the same collection."); _schedulingData = collection; _request = request; _parent = parent; _assignedNodeId = -1; _requestsWeAreBlockedBy = new Dictionary <BlockingRequestKey, SchedulableRequest>(); _requestsWeAreBlocking = new HashSet <SchedulableRequest>(); _timeRecords = new Dictionary <SchedulableRequestState, ScheduleTimeRecord>(5); _timeRecords[SchedulableRequestState.Unscheduled] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Blocked] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Yielding] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Executing] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Ready] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Completed] = new ScheduleTimeRecord(); ChangeToState(SchedulableRequestState.Unscheduled); }
/// <summary> /// Determines if the specified request is currently scheduled. /// </summary> public bool IsRequestScheduled(SchedulableRequest request) { return InternalGetScheduledRequestByGlobalRequestId(request.BuildRequest.GlobalRequestId) != null; }
/// <summary> /// Marks the parent as blocked waiting for results from a results transfer. /// </summary> private void HandleRequestBlockedOnResultsTransfer(SchedulableRequest parentRequest, List<ScheduleResponse> responses) { // Create the new request which will go to the configuration's results node. BuildRequest newRequest = new BuildRequest(parentRequest.BuildRequest.SubmissionId, BuildRequest.ResultsTransferNodeRequestId, parentRequest.BuildRequest.ConfigurationId, new string[0], null, parentRequest.BuildRequest.BuildEventContext, parentRequest.BuildRequest, parentRequest.BuildRequest.BuildRequestDataFlags); // Assign a new global request id - always different from any other. newRequest.GlobalRequestId = _nextGlobalRequestId; _nextGlobalRequestId++; // Now add the response. Send it to the node where the configuration's results are stored. When those results come back // we will update the storage location in the configuration. This is doing a bit of a run around the scheduler - we don't // create a new formal request, so we treat the blocked request as if it is still executing - this prevents any other requests // from getting onto that node and also means we don't have to do additional work to get the scheduler to understand the bizarre // case of sending a request for results from a project's own configuration (which it believes reside on the very node which // is actually requesting the results in the first place.) BuildRequestConfiguration configuration = _configCache[parentRequest.BuildRequest.ConfigurationId]; responses.Add(ScheduleResponse.CreateScheduleResponse(configuration.ResultsNodeId, newRequest, false)); TraceScheduler("Created request {0} (node request {1}) for transfer of configuration {2}'s results from node {3} to node {4}", newRequest.GlobalRequestId, newRequest.NodeRequestId, configuration.ConfigurationId, configuration.ResultsNodeId, parentRequest.AssignedNode); // The configuration's results will now be homed at the new location (once they have come back from the // original node.) configuration.ResultsNodeId = parentRequest.AssignedNode; }
/// <summary> /// Returns true if the request can be scheduled to the specified node. /// </summary> public bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId) { int requiredNodeId = GetAssignedNodeForRequestConfiguration(request.BuildRequest.ConfigurationId); return(requiredNodeId == Scheduler.InvalidNodeId || requiredNodeId == nodeId); }
/// <summary> /// Dumps the hierarchy of requests. /// </summary> private void DumpRequestHierarchy(StreamWriter file, SchedulableRequest root, int indent) { foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(root)) { DumpRequestSpec(file, child, indent, null); DumpRequestHierarchy(file, child, indent + 1); } }
/// <summary> /// Creates a new request and adds it to the system /// </summary> /// <remarks> /// New requests always go on the front of the queue, because we prefer to build the projects we just received first (depth first, absent /// any particular scheduling algorithm such as in the single-proc case.) /// </remarks> public SchedulableRequest CreateRequest(BuildRequest buildRequest, SchedulableRequest parent) { SchedulableRequest request = new SchedulableRequest(this, buildRequest, parent); request.CreationTime = EventTime; LinkedListNode<SchedulableRequest> requestNode = _unscheduledRequests.AddFirst(request); _unscheduledRequestNodesByRequest[request] = requestNode; // Update the configuration information. HashSet<SchedulableRequest> requests; if (!_configurationToRequests.TryGetValue(request.BuildRequest.ConfigurationId, out requests)) { requests = new HashSet<SchedulableRequest>(); _configurationToRequests[request.BuildRequest.ConfigurationId] = requests; } requests.Add(request); // Update the build hierarchy. if (!_buildHierarchy.ContainsKey(request)) { _buildHierarchy[request] = new List<SchedulableRequest>(8); } if (parent != null) { ErrorUtilities.VerifyThrow(_buildHierarchy.ContainsKey(parent), "Parent doesn't exist in build hierarchy for request {0}", request.BuildRequest.GlobalRequestId); _buildHierarchy[parent].Add(request); } return request; }
/// <summary> /// Marks the specified request and all of its ancestors as having aborted. /// </summary> private void MarkRequestAborted(SchedulableRequest request) { _resultsCache.AddResult(new BuildResult(request.BuildRequest, new BuildAbortedException())); // Recursively abort all of the requests we are blocking. foreach (SchedulableRequest blockedRequest in request.RequestsWeAreBlocking) { MarkRequestAborted(blockedRequest); } }
/// <summary> /// Detects a circular dependency. Throws a CircularDependencyException if one exists. Circular dependencies can occur /// under the following conditions: /// 1. If the blocking request's global request ID appears in the ancestor chain (Direct). /// 2. If a request appears in the ancestor chain and has a different global request ID but has an active target that /// matches one of the targets specified in the blocking request (Direct). /// 3. If the blocking request exists elsewhere as a blocked request with the same global request ID, and one of its children /// (recursively) matches this request's global request ID (Indirect). /// 4. If the blocking request's configuration is part of another request elsewhere which is also blocked, and that request /// is building targets this blocking request is building, and one of that blocked request's children (recursively) /// matches this request's global request ID (Indirect). /// </summary> private void DetectCircularDependency(SchedulableRequest blockingRequest) { DetectDirectCircularDependency(blockingRequest); DetectIndirectCircularDependency(blockingRequest); }
/// <summary> /// This method determines how many requests are waiting for this request, taking into account the full tree of all requests /// in all dependency chains which are waiting. /// </summary> private int ComputeClosureOfWaitingRequests(SchedulableRequest request) { int waitingRequests = 0; // In single-proc, this doesn't matter since scheduling is always 100% efficient. if (_componentHost.BuildParameters.MaxNodeCount > 1) { foreach (SchedulableRequest waitingRequest in request.RequestsWeAreBlocking) { waitingRequests++; waitingRequests += ComputeClosureOfWaitingRequests(waitingRequest); } } return waitingRequests; }
/// <summary> /// Iterates through the set of available nodes and checks whether any of them is /// capable of servicing this request or any of the requests that it is blocked /// by (regardless of whether they are currently available to do so). /// </summary> private bool RequestOrAnyItIsBlockedByCanBeServiced(SchedulableRequest request) { if (request.RequestsWeAreBlockedByCount > 0) { foreach (SchedulableRequest requestWeAreBlockedBy in request.RequestsWeAreBlockedBy) { if (RequestOrAnyItIsBlockedByCanBeServiced(requestWeAreBlockedBy)) { return true; } } // if none of the requests we are blocked by can be serviced, it doesn't matter // whether we can be serviced or not -- the reason we're blocked is because none // of the requests we are blocked by can be serviced. return false; } else { foreach (NodeInfo node in _availableNodes.Values) { if (CanScheduleRequestToNode(request, node.NodeId)) { return true; } } return false; } }
/// <summary> /// Returns true if the request can be scheduled to the specified node. /// </summary> public bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId) { int requiredNodeId = GetAssignedNodeForRequestConfiguration(request.BuildRequest.ConfigurationId); return requiredNodeId == Scheduler.InvalidNodeId || requiredNodeId == nodeId; }
/// <summary> /// Returns true if the request can be scheduled to the specified node. /// </summary> public bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId) { return(CanScheduleConfigurationToNode(request.BuildRequest.ConfigurationId, nodeId)); }
/// <summary> /// Updates the state of a request based on its desire to yield or reacquire control of its node. /// </summary> private void HandleYieldAction(SchedulableRequest parentRequest, BuildRequestBlocker blocker) { if (blocker.YieldAction == YieldAction.Yield) { // Mark the request blocked. parentRequest.Yield(blocker.TargetsInProgress); } else { // Mark the request ready. parentRequest.Reacquire(); } }
/// <summary> /// Dumps detailed information about a request. /// </summary> private void DumpRequestSpec(StreamWriter file, SchedulableRequest request, int indent, string prefix) { file.WriteLine("{0}{1}{2}: [{3}] {4}{5} ({6}){7} ({8})", new String(' ', indent * 2), (prefix == null) ? "" : prefix, request.BuildRequest.GlobalRequestId, _schedulingData.GetAssignedNodeForRequestConfiguration(request.BuildRequest.ConfigurationId), _schedulingData.IsRequestScheduled(request) ? "RUNNING " : "", request.State, request.BuildRequest.ConfigurationId, _configCache[request.BuildRequest.ConfigurationId].ProjectFullPath, String.Join(", ", request.BuildRequest.Targets.ToArray())); }
/// <summary> /// Dumps the state of a request. /// </summary> private void DumpRequestState(StreamWriter file, SchedulableRequest request, int indent) { DumpRequestSpec(file, request, indent, null); if (request.RequestsWeAreBlockedByCount > 0) { foreach (SchedulableRequest blockingRequest in request.RequestsWeAreBlockedBy) { DumpRequestSpec(file, blockingRequest, indent + 1, "!"); } } if (request.RequestsWeAreBlockingCount > 0) { foreach (SchedulableRequest blockedRequest in request.RequestsWeAreBlocking) { DumpRequestSpec(file, blockedRequest, indent + 1, ">"); } } }
/// <summary> /// Attempts to get results from the cache for this request. If results are available, reports them to the /// correct node. If that action causes the parent to become ready and its node is idle, the parent is /// resumed. /// </summary> private void ResolveRequestFromCacheAndResumeIfPossible(SchedulableRequest request, List<ScheduleResponse> responses) { int nodeForResults = (request.Parent != null) ? request.Parent.AssignedNode : InvalidNodeId; // Do we already have results? If so, just return them. ScheduleResponse response = TrySatisfyRequestFromCache(nodeForResults, request.BuildRequest, skippedResultsAreOK: false); if (response != null) { if (response.Action == ScheduleActionType.SubmissionComplete) { ErrorUtilities.VerifyThrow(request.Parent == null, "Unexpectedly generated a SubmissionComplete response for a request which is not top-level."); LogRequestHandledFromCache(request.BuildRequest, response.BuildResult); // This was root request, we can report submission complete. responses.Add(ScheduleResponse.CreateSubmissionCompleteResponse(response.BuildResult)); if (response.BuildResult.OverallResult != BuildResultCode.Failure) { WriteSchedulingPlan(response.BuildResult.SubmissionId); } } else { LogRequestHandledFromCache(request.BuildRequest, response.Unblocker.Result); request.Complete(response.Unblocker.Result); TraceScheduler("Reporting results for request {0} with parent {1} to node {2} from cache.", request.BuildRequest.GlobalRequestId, request.BuildRequest.ParentGlobalRequestId, response.NodeId); if (response.NodeId != InvalidNodeId) { responses.Add(response); } // Is the node we are reporting to idle? If so, does reporting this result allow it to proceed with work? if (!_schedulingData.IsNodeWorking(response.NodeId)) { ResumeReadyRequestIfAny(response.NodeId, responses); } } } }
/// <summary> /// Updates the state of the specified request. /// </summary> public void UpdateFromState(SchedulableRequest request, SchedulableRequestState previousState) { // Remove from its old collection switch (previousState) { case SchedulableRequestState.Blocked: _blockedRequests.Remove(request.BuildRequest.GlobalRequestId); break; case SchedulableRequestState.Yielding: _yieldingRequests.Remove(request.BuildRequest.GlobalRequestId); break; case SchedulableRequestState.Completed: ErrorUtilities.ThrowInternalError("Should not be updating a request after it has reached the Completed state."); break; case SchedulableRequestState.Executing: _executingRequests.Remove(request.BuildRequest.GlobalRequestId); _executingRequestByNode[request.AssignedNode] = null; break; case SchedulableRequestState.Ready: _readyRequests.Remove(request.BuildRequest.GlobalRequestId); _readyRequestsByNode[request.AssignedNode].Remove(request); break; case SchedulableRequestState.Unscheduled: LinkedListNode<SchedulableRequest> requestNode = _unscheduledRequestNodesByRequest[request]; _unscheduledRequestNodesByRequest.Remove(request); _unscheduledRequests.Remove(requestNode); if (request.State != SchedulableRequestState.Completed) { // Map the request to the node. HashSet<SchedulableRequest> requestsAssignedToNode; if (!_scheduledRequestsByNode.TryGetValue(request.AssignedNode, out requestsAssignedToNode)) { requestsAssignedToNode = new HashSet<SchedulableRequest>(); _scheduledRequestsByNode[request.AssignedNode] = requestsAssignedToNode; } ErrorUtilities.VerifyThrow(!requestsAssignedToNode.Contains(request), "Request {0} is already scheduled to node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode); requestsAssignedToNode.Add(request); // Map the configuration to the node. HashSet<int> configurationsAssignedToNode; if (!_configurationsByNode.TryGetValue(request.AssignedNode, out configurationsAssignedToNode)) { configurationsAssignedToNode = new HashSet<int>(); _configurationsByNode[request.AssignedNode] = configurationsAssignedToNode; } if (!configurationsAssignedToNode.Contains(request.BuildRequest.ConfigurationId)) { configurationsAssignedToNode.Add(request.BuildRequest.ConfigurationId); } } break; } // Add it to its new location switch (request.State) { case SchedulableRequestState.Blocked: ErrorUtilities.VerifyThrow(!_blockedRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already blocked!"); _blockedRequests[request.BuildRequest.GlobalRequestId] = request; break; case SchedulableRequestState.Yielding: ErrorUtilities.VerifyThrow(!_yieldingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already yielded!"); _yieldingRequests[request.BuildRequest.GlobalRequestId] = request; break; case SchedulableRequestState.Completed: ErrorUtilities.VerifyThrow(_configurationToRequests.ContainsKey(request.BuildRequest.ConfigurationId), "Configuration {0} never had requests assigned to it.", request.BuildRequest.ConfigurationId); ErrorUtilities.VerifyThrow(_configurationToRequests[request.BuildRequest.ConfigurationId].Count > 0, "Configuration {0} has no requests assigned to it.", request.BuildRequest.ConfigurationId); _configurationToRequests[request.BuildRequest.ConfigurationId].Remove(request); if (_scheduledRequestsByNode.ContainsKey(request.AssignedNode)) { _scheduledRequestsByNode[request.AssignedNode].Remove(request); } request.EndTime = EventTime; break; case SchedulableRequestState.Executing: ErrorUtilities.VerifyThrow(!_executingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already executing!"); ErrorUtilities.VerifyThrow(!_executingRequestByNode.ContainsKey(request.AssignedNode) || _executingRequestByNode[request.AssignedNode] == null, "Node {0} is currently executing a request.", request.AssignedNode); _executingRequests[request.BuildRequest.GlobalRequestId] = request; _executingRequestByNode[request.AssignedNode] = request; _configurationToNode[request.BuildRequest.ConfigurationId] = request.AssignedNode; if (previousState == SchedulableRequestState.Unscheduled) { request.StartTime = EventTime; } break; case SchedulableRequestState.Ready: ErrorUtilities.VerifyThrow(!_readyRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already ready!"); _readyRequests[request.BuildRequest.GlobalRequestId] = request; HashSet<SchedulableRequest> readyRequestsOnNode; if (!_readyRequestsByNode.TryGetValue(request.AssignedNode, out readyRequestsOnNode)) { readyRequestsOnNode = new HashSet<SchedulableRequest>(); _readyRequestsByNode[request.AssignedNode] = readyRequestsOnNode; } ErrorUtilities.VerifyThrow(!readyRequestsOnNode.Contains(request), "Request with global id {0} is already marked as ready on node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode); readyRequestsOnNode.Add(request); break; case SchedulableRequestState.Unscheduled: ErrorUtilities.ThrowInternalError("Request with global id {0} cannot transition to the Unscheduled state", request.BuildRequest.GlobalRequestId); break; } _buildEvents.Add(new SchedulingEvent(EventTime, request, previousState, request.State)); }
/// <summary> /// Marks this request as being blocked by the specified request. Establishes the correct relationships between the requests. /// </summary> /// <param name="blockingRequest">The request which is blocking this one.</param> /// <param name="activeTargets">The list of targets this request was currently building at the time it became blocked.</param> public void BlockByRequest(SchedulableRequest blockingRequest, string[] activeTargets) { VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Blocked, SchedulableRequestState.Executing }); ErrorUtilities.VerifyThrowArgumentNull(blockingRequest, "blockingRequest"); ErrorUtilities.VerifyThrowArgumentNull(activeTargets, "activeTargets"); // Note that the blocking request will typically be our parent UNLESS it is a request we blocked on because it was executing a target we wanted to execute. // Thus, we do not assert the parent-child relationship here. BlockingRequestKey key = new BlockingRequestKey(blockingRequest.BuildRequest); ErrorUtilities.VerifyThrow(!_requestsWeAreBlockedBy.ContainsKey(key), "We are already blocked by this request."); ErrorUtilities.VerifyThrow(!blockingRequest._requestsWeAreBlocking.Contains(this), "The blocking request thinks it is already blocking us."); // This method is only called when a request reports that it is blocked on other requests. If the request is being blocked by a brand new // request, that request will be unscheduled. If this request is blocked by an in-progress request which was executing a target it needed // to also execute, then that request is not unscheduled (because it was running on the node) and it is not executing (because this condition // can only occur against requests which are executing on the same node and since the request which called this method is the one currently // executing on that node, that means the request it is blocked by must either be itself blocked or ready.) blockingRequest.VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Yielding, SchedulableRequestState.Blocked, SchedulableRequestState.Ready, SchedulableRequestState.Unscheduled }); // Update our list of active targets. This has to be done before we detect circular dependencies because we use this information to detect // re-entrancy circular dependencies. _activeTargetsWhenBlocked = activeTargets; DetectCircularDependency(blockingRequest); _requestsWeAreBlockedBy[key] = blockingRequest; blockingRequest._requestsWeAreBlocking.Add(this); ChangeToState(SchedulableRequestState.Blocked); }
/// <summary> /// Marks the request as being blocked by another request which is currently building a target whose results we need to proceed. /// </summary> private void HandleRequestBlockedOnInProgressTarget(SchedulableRequest blockedRequest, BuildRequestBlocker blocker) { ErrorUtilities.VerifyThrowArgumentNull(blockedRequest, "blockedRequest"); ErrorUtilities.VerifyThrowArgumentNull(blocker, "blocker"); // We are blocked on an in-progress request building a target whose results we need. SchedulableRequest blockingRequest = _schedulingData.GetScheduledRequest(blocker.BlockingRequestId); // The request we blocked on couldn't have been executing (because we are) so it must either be yielding (which is ok because // it isn't modifying its own state, just running a background process), ready, or still blocked. blockingRequest.VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Yielding, SchedulableRequestState.Ready, SchedulableRequestState.Blocked }); blockedRequest.BlockByRequest(blockingRequest, blocker.TargetsInProgress); }
/// <summary> /// Assigns the specified request to the specified node. /// </summary> private void AssignUnscheduledRequestToNode(SchedulableRequest request, int nodeId, List<ScheduleResponse> responses) { ErrorUtilities.VerifyThrowArgumentNull(request, "request"); ErrorUtilities.VerifyThrowArgumentNull(responses, "responses"); ErrorUtilities.VerifyThrow(nodeId != InvalidNodeId, "Invalid node id specified."); // Currently we cannot move certain kinds of traversals (notably solution metaprojects) to other nodes because // they only have a ProjectInstance representation, and besides these kinds of projects build very quickly // and produce more references (more work to do.) This just verifies we do not attempt to send a traversal to // an out-of-proc node because doing so is inefficient and presently will cause the engine to fail on the remote // node because these projects cannot be found. ErrorUtilities.VerifyThrow(nodeId == InProcNodeId || _forceAffinityOutOfProc || !IsTraversalRequest(request.BuildRequest), "Can't assign traversal request to out-of-proc node!"); request.VerifyState(SchedulableRequestState.Unscheduled); // Determine if this node has seen our configuration before. If not, we must send it along with this request. bool mustSendConfigurationToNode = _availableNodes[nodeId].AssignConfiguration(request.BuildRequest.ConfigurationId); // If this is the first time this configuration has been assigned to a node, we will mark the configuration with the assigned node // indicating that the master set of results is located there. Should we ever need to move the results, we will know where to find them. BuildRequestConfiguration config = _configCache[request.BuildRequest.ConfigurationId]; if (config.ResultsNodeId == InvalidNodeId) { config.ResultsNodeId = nodeId; } ErrorUtilities.VerifyThrow(config.ResultsNodeId != InvalidNodeId, "Configuration's results node is not set."); responses.Add(ScheduleResponse.CreateScheduleResponse(nodeId, request.BuildRequest, mustSendConfigurationToNode)); TraceScheduler("Executing request {0} on node {1} with parent {2}", request.BuildRequest.GlobalRequestId, nodeId, (request.Parent == null) ? -1 : request.Parent.BuildRequest.GlobalRequestId); request.ResumeExecution(nodeId); }
/// <summary> /// Returns true if a request can be scheduled to a node, false otherwise. /// </summary> private bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId) { if (_schedulingData.CanScheduleRequestToNode(request, nodeId)) { NodeAffinity affinity = GetNodeAffinityForRequest(request.BuildRequest); bool result = _availableNodes[nodeId].CanServiceRequestWithAffinity(affinity); return result; } return false; }
/// <summary> /// Marks the request as being blocked by new requests whose results we must get before we can proceed. /// </summary> private void HandleRequestBlockedByNewRequests(SchedulableRequest parentRequest, BuildRequestBlocker blocker, List<ScheduleResponse> responses) { ErrorUtilities.VerifyThrowArgumentNull(blocker, "blocker"); ErrorUtilities.VerifyThrowArgumentNull(responses, "responses"); // The request is waiting on new requests. bool abortRequestBatch = false; Stack<BuildRequest> requestsToAdd = new Stack<BuildRequest>(blocker.BuildRequests.Length); foreach (BuildRequest request in blocker.BuildRequests) { // Assign a global request id to this request. if (request.GlobalRequestId == BuildRequest.InvalidGlobalRequestId) { AssignGlobalRequestId(request); } int nodeForResults = (parentRequest == null) ? InvalidNodeId : parentRequest.AssignedNode; TraceScheduler("Received request {0} (node request {1}) with parent {2} from node {3}", request.GlobalRequestId, request.NodeRequestId, request.ParentGlobalRequestId, nodeForResults); // First, determine if we have already built this request and have results for it. If we do, we prepare the responses for it // directly here. We COULD simply report these as blocking the parent request and let the scheduler pick them up later when the parent // comes back up as schedulable, but we prefer to send the results back immediately so this request can (potentially) continue uninterrupted. ScheduleResponse response = TrySatisfyRequestFromCache(nodeForResults, request, skippedResultsAreOK: false); if (null != response) { TraceScheduler("Request {0} (node request {1}) satisfied from the cache.", request.GlobalRequestId, request.NodeRequestId); // BuildResult result = (response.Action == ScheduleActionType.Unblock) ? response.Unblocker.Results[0] : response.BuildResult; LogRequestHandledFromCache(request, response.BuildResult); responses.Add(response); // If we wish to implement an algorithm where the first failing request aborts the remaining request, check for // overall result being failure rather than just circular dependency. Sync with BasicScheduler.ReportResult and // BuildRequestEntry.ReportResult. if (response.BuildResult.CircularDependency) { abortRequestBatch = true; } } else { // Ensure there is no affinity mismatch between this request and a previous request of the same configuration. NodeAffinity requestAffinity = GetNodeAffinityForRequest(request); NodeAffinity existingRequestAffinity = NodeAffinity.Any; if (requestAffinity != NodeAffinity.Any) { bool affinityMismatch = false; int assignedNodeId = _schedulingData.GetAssignedNodeForRequestConfiguration(request.ConfigurationId); if (assignedNodeId != Scheduler.InvalidNodeId) { if (!_availableNodes[assignedNodeId].CanServiceRequestWithAffinity(GetNodeAffinityForRequest(request))) { // This request's configuration has already been assigned to a node which cannot service this affinity. if (_schedulingData.GetRequestsAssignedToConfigurationCount(request.ConfigurationId) == 0) { // If there are no other requests already scheduled for that configuration, we can safely reassign. _schedulingData.UnassignNodeForRequestConfiguration(request.ConfigurationId); } else { existingRequestAffinity = (_availableNodes[assignedNodeId].ProviderType == NodeProviderType.InProc) ? NodeAffinity.InProc : NodeAffinity.OutOfProc; affinityMismatch = true; } } } else if (_schedulingData.GetRequestsAssignedToConfigurationCount(request.ConfigurationId) > 0) { // Would any other existing requests for this configuration mismatch? foreach (SchedulableRequest existingRequest in _schedulingData.GetRequestsAssignedToConfiguration(request.ConfigurationId)) { existingRequestAffinity = GetNodeAffinityForRequest(existingRequest.BuildRequest); if (existingRequestAffinity != NodeAffinity.Any && existingRequestAffinity != requestAffinity) { // The existing request has an affinity which doesn't match this one, so this one could never be scheduled. affinityMismatch = true; break; } } } if (affinityMismatch) { BuildResult result = new BuildResult(request, new InvalidOperationException(ResourceUtilities.FormatResourceString("AffinityConflict", requestAffinity, existingRequestAffinity))); response = GetResponseForResult(nodeForResults, request, result); responses.Add(response); continue; } } // Now add the requests so they would naturally be picked up by the scheduler in the order they were issued, // but before any other requests in the list. This makes us prefer a depth-first traversal. requestsToAdd.Push(request); } } // Now add any unassigned build requests. if (!abortRequestBatch) { if (requestsToAdd.Count == 0) { // All of the results are being reported directly from the cache (from above), so this request can continue on its merry way. if (parentRequest != null) { // responses.Add(new ScheduleResponse(parentRequest.AssignedNode, new BuildRequestUnblocker(parentRequest.BuildRequest.GlobalRequestId))); responses.Add(ScheduleResponse.CreateResumeExecutionResponse(parentRequest.AssignedNode, parentRequest.BuildRequest.GlobalRequestId)); } } else { while (requestsToAdd.Count > 0) { BuildRequest requestToAdd = requestsToAdd.Pop(); SchedulableRequest blockingRequest = _schedulingData.CreateRequest(requestToAdd, parentRequest); if (parentRequest != null) { parentRequest.BlockByRequest(blockingRequest, blocker.TargetsInProgress); } } } } }
/// <summary> /// Detects a circular dependency where the request which is about to block us is already blocked by us, usually as a result /// of it having been previously scheduled in a multiproc scenario, but before this request was able to execute. /// </summary> /// <remarks> /// Let A be 'this' project and B be 'blockingRequest' (the request which is going to block A.) /// An indirect circular dependency exists if there is a dependency path from B to A. If there is no /// existing blocked request B' with the same global request id as B, then there can be no path from B to A because B is a brand new /// request with no other dependencies. If there is an existing blocked request B' with the same global request ID as B, then we /// walk the set of dependencies recursively searching for A. If A is found, we have a circular dependency. /// </remarks> private void DetectIndirectCircularDependency(SchedulableRequest blockingRequest) { // If there is already a blocked request which has the same configuration id as the blocking request and that blocked request is (recursively) // waiting on this request, then that is an indirect circular dependency. SchedulableRequest alternateRequest = _schedulingData.GetBlockedRequestIfAny(blockingRequest.BuildRequest.GlobalRequestId); if (alternateRequest == null) { return; } Stack<SchedulableRequest> requestsToEvaluate = new Stack<SchedulableRequest>(16); HashSet<SchedulableRequest> evaluatedRequests = new HashSet<SchedulableRequest>(); requestsToEvaluate.Push(alternateRequest); while (requestsToEvaluate.Count > 0) { SchedulableRequest requestToEvaluate = requestsToEvaluate.Pop(); // If we make it to a child which is us, then it's a circular dependency. if (requestToEvaluate.BuildRequest.GlobalRequestId == this.BuildRequest.GlobalRequestId) { ThrowIndirectCircularDependency(blockingRequest, requestToEvaluate); } evaluatedRequests.Add(requestToEvaluate); // If the request is not scheduled, it's possible that is because it's been scheduled elsewhere and is blocked. // Follow that path if it exists. if (requestToEvaluate.State == SchedulableRequestState.Unscheduled) { requestToEvaluate = _schedulingData.GetBlockedRequestIfAny(requestToEvaluate.BuildRequest.GlobalRequestId); // If there was no scheduled request to evaluate, move on. if (requestToEvaluate == null || evaluatedRequests.Contains(requestToEvaluate)) { continue; } } // This request didn't cause a circular dependency, check its children. foreach (SchedulableRequest childRequest in requestToEvaluate.RequestsWeAreBlockedBy) { if (!evaluatedRequests.Contains(childRequest)) { requestsToEvaluate.Push(childRequest); } } } }
/// <summary> /// Recursively dumps the build information for the specified hierarchy /// </summary> private void WriteRecursiveSummary(ILoggingService loggingService, BuildEventContext context, int submissionId, SchedulableRequest request, int level, bool useConfigurations, bool isLastChild) { int postPad = Math.Max(20 /* field width */ - (2 * level) /* spacing for hierarchy lines */ - 3 /* length allocated for config/request id */, 0); StringBuilder prePadString = new StringBuilder(2 * level); if (level != 0) { int levelsToPad = level; if (isLastChild) { levelsToPad--; } while (levelsToPad > 0) { prePadString.Append("| "); levelsToPad--; } if (isLastChild) { prePadString.Append(@". "); } } loggingService.LogComment ( context, MessageImportance.Normal, "BuildHierarchyEntry", prePadString.ToString(), useConfigurations ? request.BuildRequest.ConfigurationId : request.BuildRequest.GlobalRequestId, new String(' ', postPad), String.Format(CultureInfo.InvariantCulture, "{0:0.000}", request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalSeconds), String.Format(CultureInfo.InvariantCulture, "{0:0.000}", request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalSeconds + request.GetTimeSpentInState(SchedulableRequestState.Blocked).TotalSeconds + request.GetTimeSpentInState(SchedulableRequestState.Ready).TotalSeconds), _configCache[request.BuildRequest.ConfigurationId].ProjectFullPath, String.Join(", ", request.BuildRequest.Targets.ToArray()) ); List<SchedulableRequest> childRequests = new List<SchedulableRequest>(_schedulingData.GetRequestsByHierarchy(request)); childRequests.Sort(delegate (SchedulableRequest left, SchedulableRequest right) { if (left.StartTime < right.StartTime) { return -1; } else if (left.StartTime > right.StartTime) { return 1; } return 0; }); for (int i = 0; i < childRequests.Count; i++) { SchedulableRequest childRequest = childRequests[i]; WriteRecursiveSummary(loggingService, context, submissionId, childRequest, level + 1, useConfigurations, i == childRequests.Count - 1); } }
/// <summary> /// Recursively accumulates the amount of time spent in each configuration. /// </summary> private void RecursiveAccumulateConfigurationTimes(SchedulableRequest request, Dictionary<int, double> accumulatedTimeByConfiguration) { double accumulatedTime; // NOTE: Do we want to count it each time the config appears in the hierarchy? This will inflate the // cost of frequently referenced configurations. accumulatedTimeByConfiguration.TryGetValue(request.BuildRequest.ConfigurationId, out accumulatedTime); accumulatedTimeByConfiguration[request.BuildRequest.ConfigurationId] = accumulatedTime + request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalMilliseconds; foreach (SchedulableRequest childRequest in _schedulingData.GetRequestsByHierarchy(request)) { RecursiveAccumulateConfigurationTimes(childRequest, accumulatedTimeByConfiguration); } }
/// <summary> /// Detects a circular dependency where the blocking request is in our direct ancestor chain. /// </summary> private void DetectDirectCircularDependency(SchedulableRequest blockingRequest) { // A circular dependency occurs when this project (or any of its ancestors) has the same global request id as the // blocking request. List<SchedulableRequest> ancestors = new List<SchedulableRequest>(16); SchedulableRequest currentRequest = this; do { ancestors.Add(currentRequest); if (currentRequest.BuildRequest.GlobalRequestId == blockingRequest.BuildRequest.GlobalRequestId) { // We are directly conflicting with an instance of ourselves. CleanupForCircularDependencyAndThrow(blockingRequest, ancestors); } currentRequest = currentRequest.Parent; } while (currentRequest != null); }
/// <summary> /// Writes out all of the dependencies for a specified request, recursively. /// </summary> private void RecursiveWriteDependencies(StreamWriter file, SchedulableRequest request) { file.Write(request.BuildRequest.ConfigurationId); foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request)) { file.Write(" {0}", child.BuildRequest.ConfigurationId); } file.WriteLine(); foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request)) { RecursiveWriteDependencies(file, child); } }
/// <summary> /// Retrieves a set of build requests which have the specified parent. If root is null, this will retrieve all of the /// top-level requests. /// </summary> public IEnumerable<SchedulableRequest> GetRequestsByHierarchy(SchedulableRequest root) { if (root == null) { // Retrieve all requests which are roots of the tree. List<SchedulableRequest> roots = new List<SchedulableRequest>(); foreach (SchedulableRequest key in _buildHierarchy.Keys) { if (key.Parent == null) { roots.Add(key); } } return roots; } return _buildHierarchy[root]; }
/// <summary> /// Constructor. /// </summary> public SchedulableRequest(SchedulingData collection, BuildRequest request, SchedulableRequest parent) { ErrorUtilities.VerifyThrowArgumentNull(collection, "collection"); ErrorUtilities.VerifyThrowArgumentNull(request, "request"); ErrorUtilities.VerifyThrow((parent == null) || (parent._schedulingData == collection), "Parent request does not belong to the same collection."); _schedulingData = collection; _request = request; _parent = parent; _assignedNodeId = -1; _requestsWeAreBlockedBy = new Dictionary<BlockingRequestKey, SchedulableRequest>(); _requestsWeAreBlocking = new HashSet<SchedulableRequest>(); _timeRecords = new Dictionary<SchedulableRequestState, ScheduleTimeRecord>(5); _timeRecords[SchedulableRequestState.Unscheduled] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Blocked] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Yielding] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Executing] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Ready] = new ScheduleTimeRecord(); _timeRecords[SchedulableRequestState.Completed] = new ScheduleTimeRecord(); ChangeToState(SchedulableRequestState.Unscheduled); }
/// <summary> /// Determines if the specified request is currently scheduled. /// </summary> public bool IsRequestScheduled(SchedulableRequest request) { return(InternalGetScheduledRequestByGlobalRequestId(request.BuildRequest.GlobalRequestId) != null); }
/// <summary> /// Detects a circular dependency. Throws a CircularDependencyException if one exists. Circular dependencies can occur /// under the following conditions: /// 1. If the blocking request's global request ID appears in the ancestor chain (Direct). /// 2. If a request appears in the ancestor chain and has a different global request ID but has an active target that /// matches one of the targets specified in the blocking request (Direct). /// 3. If the blocking request exists elsewhere as a blocked request with the same global request ID, and one of its children /// (recursively) matches this request's global request ID (Indirect). /// 4. If the blocking request's configuration is part of another request elsewhere which is also blocked, and that request /// is building targets this blocking request is building, and one of that blocked request's children (recursively) /// matches this request's global request ID (Indirect). /// </summary> private void DetectCircularDependency(SchedulableRequest blockingRequest) { DetectDirectCircularDependency(blockingRequest); DetectIndirectCircularDependency(blockingRequest); }
/// <summary> /// Constructor. /// </summary> public SchedulingEvent(DateTime eventTime, SchedulableRequest request, SchedulableRequestState oldState, SchedulableRequestState newState) { _eventTime = eventTime; _request = request; _oldState = oldState; _newState = newState; }
/// <summary> /// Build our ancestor list then throw the circular dependency error. /// </summary> private void ThrowIndirectCircularDependency(SchedulableRequest blockingRequest, SchedulableRequest requestToEvaluate) { // We found a request which has the same global request ID as us in a chain which leads from the (already blocked) request // which is trying to block us. Calculate its list of ancestors by walking up the parent list. List<SchedulableRequest> ancestors = new List<SchedulableRequest>(16); while (requestToEvaluate.Parent != null) { ancestors.Add(requestToEvaluate.Parent); requestToEvaluate = requestToEvaluate.Parent; } ancestors.Reverse(); // Because the list should be in the order from root to child. CleanupForCircularDependencyAndThrow(blockingRequest, ancestors); }
/// <summary> /// Updates the state of the specified request. /// </summary> public void UpdateFromState(SchedulableRequest request, SchedulableRequestState previousState) { // Remove from its old collection switch (previousState) { case SchedulableRequestState.Blocked: _blockedRequests.Remove(request.BuildRequest.GlobalRequestId); break; case SchedulableRequestState.Yielding: _yieldingRequests.Remove(request.BuildRequest.GlobalRequestId); break; case SchedulableRequestState.Completed: ErrorUtilities.ThrowInternalError("Should not be updating a request after it has reached the Completed state."); break; case SchedulableRequestState.Executing: _executingRequests.Remove(request.BuildRequest.GlobalRequestId); _executingRequestByNode[request.AssignedNode] = null; break; case SchedulableRequestState.Ready: _readyRequests.Remove(request.BuildRequest.GlobalRequestId); _readyRequestsByNode[request.AssignedNode].Remove(request); break; case SchedulableRequestState.Unscheduled: LinkedListNode <SchedulableRequest> requestNode = _unscheduledRequestNodesByRequest[request]; _unscheduledRequestNodesByRequest.Remove(request); _unscheduledRequests.Remove(requestNode); if (request.State != SchedulableRequestState.Completed) { // Map the request to the node. HashSet <SchedulableRequest> requestsAssignedToNode; if (!_scheduledRequestsByNode.TryGetValue(request.AssignedNode, out requestsAssignedToNode)) { requestsAssignedToNode = new HashSet <SchedulableRequest>(); _scheduledRequestsByNode[request.AssignedNode] = requestsAssignedToNode; } ErrorUtilities.VerifyThrow(!requestsAssignedToNode.Contains(request), "Request {0} is already scheduled to node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode); requestsAssignedToNode.Add(request); // Map the configuration to the node. HashSet <int> configurationsAssignedToNode; if (!_configurationsByNode.TryGetValue(request.AssignedNode, out configurationsAssignedToNode)) { configurationsAssignedToNode = new HashSet <int>(); _configurationsByNode[request.AssignedNode] = configurationsAssignedToNode; } if (!configurationsAssignedToNode.Contains(request.BuildRequest.ConfigurationId)) { configurationsAssignedToNode.Add(request.BuildRequest.ConfigurationId); } } break; } // Add it to its new location switch (request.State) { case SchedulableRequestState.Blocked: ErrorUtilities.VerifyThrow(!_blockedRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already blocked!"); _blockedRequests[request.BuildRequest.GlobalRequestId] = request; break; case SchedulableRequestState.Yielding: ErrorUtilities.VerifyThrow(!_yieldingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already yielded!"); _yieldingRequests[request.BuildRequest.GlobalRequestId] = request; break; case SchedulableRequestState.Completed: ErrorUtilities.VerifyThrow(_configurationToRequests.ContainsKey(request.BuildRequest.ConfigurationId), "Configuration {0} never had requests assigned to it.", request.BuildRequest.ConfigurationId); ErrorUtilities.VerifyThrow(_configurationToRequests[request.BuildRequest.ConfigurationId].Count > 0, "Configuration {0} has no requests assigned to it.", request.BuildRequest.ConfigurationId); _configurationToRequests[request.BuildRequest.ConfigurationId].Remove(request); if (_scheduledRequestsByNode.ContainsKey(request.AssignedNode)) { _scheduledRequestsByNode[request.AssignedNode].Remove(request); } request.EndTime = EventTime; break; case SchedulableRequestState.Executing: ErrorUtilities.VerifyThrow(!_executingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already executing!"); ErrorUtilities.VerifyThrow(!_executingRequestByNode.ContainsKey(request.AssignedNode) || _executingRequestByNode[request.AssignedNode] == null, "Node {0} is currently executing a request.", request.AssignedNode); _executingRequests[request.BuildRequest.GlobalRequestId] = request; _executingRequestByNode[request.AssignedNode] = request; _configurationToNode[request.BuildRequest.ConfigurationId] = request.AssignedNode; if (previousState == SchedulableRequestState.Unscheduled) { request.StartTime = EventTime; } break; case SchedulableRequestState.Ready: ErrorUtilities.VerifyThrow(!_readyRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already ready!"); _readyRequests[request.BuildRequest.GlobalRequestId] = request; HashSet <SchedulableRequest> readyRequestsOnNode; if (!_readyRequestsByNode.TryGetValue(request.AssignedNode, out readyRequestsOnNode)) { readyRequestsOnNode = new HashSet <SchedulableRequest>(); _readyRequestsByNode[request.AssignedNode] = readyRequestsOnNode; } ErrorUtilities.VerifyThrow(!readyRequestsOnNode.Contains(request), "Request with global id {0} is already marked as ready on node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode); readyRequestsOnNode.Add(request); break; case SchedulableRequestState.Unscheduled: ErrorUtilities.ThrowInternalError("Request with global id {0} cannot transition to the Unscheduled state", request.BuildRequest.GlobalRequestId); break; } _buildEvents.Add(new SchedulingEvent(EventTime, request, previousState, request.State)); }
/// <summary> /// Removes associations with all blocking requests and throws an exception. /// </summary> private void CleanupForCircularDependencyAndThrow(SchedulableRequest requestCausingFailure, List<SchedulableRequest> ancestors) { if (_requestsWeAreBlockedBy.Count != 0) { List<SchedulableRequest> tempRequests = new List<SchedulableRequest>(_requestsWeAreBlockedBy.Values); foreach (SchedulableRequest requestWeAreBlockedBy in tempRequests) { BlockingRequestKey key = new BlockingRequestKey(requestWeAreBlockedBy.BuildRequest); DisconnectRequestWeAreBlockedBy(key); } } else { ChangeToState(SchedulableRequestState.Ready); } _activeTargetsWhenBlocked = null; // The blocking request itself is no longer valid if it was unscheduled. if (requestCausingFailure.State == SchedulableRequestState.Unscheduled) { requestCausingFailure.Delete(); } throw new SchedulerCircularDependencyException(requestCausingFailure.BuildRequest, ancestors); }
/// <summary> /// Gets the name of the plan file for a specified submission. /// </summary> private string GetPlanName(SchedulableRequest rootRequest) { if (rootRequest == null) { return null; } return _configCache[rootRequest.BuildRequest.ConfigurationId].ProjectFullPath + ".buildplan"; }