/// <summary>
/// Constructor.
/// </summary>
public SchedulingEvent(DateTime eventTime, SchedulableRequest request, SchedulableRequestState oldState, SchedulableRequestState newState)
{
_eventTime = eventTime;
_request = request;
_oldState = oldState;
_newState = newState;
}
/// <summary>
/// Retrieves a request which has been assigned to a node and is in the executing, blocked or ready states.
/// </summary>
public SchedulableRequest GetScheduledRequest(int globalRequestId)
{
SchedulableRequest returnValue = InternalGetScheduledRequestByGlobalRequestId(globalRequestId);
ErrorUtilities.VerifyThrow(returnValue != null, "Global Request Id {0} has not been assigned and cannot be retrieved.", globalRequestId);
return(returnValue);
}
/// <summary>
/// Marks this request as being blocked by the specified request. Establishes the correct relationships between the requests.
/// </summary>
/// <param name="blockingRequest">The request which is blocking this one.</param>
/// <param name="activeTargets">The list of targets this request was currently building at the time it became blocked.</param>
public void BlockByRequest(SchedulableRequest blockingRequest, string[] activeTargets)
{
VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Blocked, SchedulableRequestState.Executing });
ErrorUtilities.VerifyThrowArgumentNull(blockingRequest, "blockingRequest");
ErrorUtilities.VerifyThrowArgumentNull(activeTargets, "activeTargets");
// Note that the blocking request will typically be our parent UNLESS it is a request we blocked on because it was executing a target we wanted to execute.
// Thus, we do not assert the parent-child relationship here.
BlockingRequestKey key = new BlockingRequestKey(blockingRequest.BuildRequest);
ErrorUtilities.VerifyThrow(!_requestsWeAreBlockedBy.ContainsKey(key), "We are already blocked by this request.");
ErrorUtilities.VerifyThrow(!blockingRequest._requestsWeAreBlocking.Contains(this), "The blocking request thinks it is already blocking us.");
// This method is only called when a request reports that it is blocked on other requests. If the request is being blocked by a brand new
// request, that request will be unscheduled. If this request is blocked by an in-progress request which was executing a target it needed
// to also execute, then that request is not unscheduled (because it was running on the node) and it is not executing (because this condition
// can only occur against requests which are executing on the same node and since the request which called this method is the one currently
// executing on that node, that means the request it is blocked by must either be itself blocked or ready.)
blockingRequest.VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Yielding, SchedulableRequestState.Blocked, SchedulableRequestState.Ready, SchedulableRequestState.Unscheduled });
// Update our list of active targets. This has to be done before we detect circular dependencies because we use this information to detect
// re-entrancy circular dependencies.
_activeTargetsWhenBlocked = activeTargets;
DetectCircularDependency(blockingRequest);
_requestsWeAreBlockedBy[key] = blockingRequest;
blockingRequest._requestsWeAreBlocking.Add(this);
ChangeToState(SchedulableRequestState.Blocked);
}
/// <summary>
/// Removes associations with all blocking requests and throws an exception.
/// </summary>
private void CleanupForCircularDependencyAndThrow(SchedulableRequest requestCausingFailure, List <SchedulableRequest> ancestors)
{
if (_requestsWeAreBlockedBy.Count != 0)
{
List <SchedulableRequest> tempRequests = new List <SchedulableRequest>(_requestsWeAreBlockedBy.Values);
foreach (SchedulableRequest requestWeAreBlockedBy in tempRequests)
{
BlockingRequestKey key = new BlockingRequestKey(requestWeAreBlockedBy.BuildRequest);
DisconnectRequestWeAreBlockedBy(key);
}
}
else
{
ChangeToState(SchedulableRequestState.Ready);
}
_activeTargetsWhenBlocked = null;
// The blocking request itself is no longer valid if it was unscheduled.
if (requestCausingFailure.State == SchedulableRequestState.Unscheduled)
{
requestCausingFailure.Delete();
}
throw new SchedulerCircularDependencyException(requestCausingFailure.BuildRequest, ancestors);
}
/// <summary>
/// Creates a new request and adds it to the system
/// </summary>
/// <remarks>
/// New requests always go on the front of the queue, because we prefer to build the projects we just received first (depth first, absent
/// any particular scheduling algorithm such as in the single-proc case.)
/// </remarks>
public SchedulableRequest CreateRequest(BuildRequest buildRequest, SchedulableRequest parent)
{
SchedulableRequest request = new SchedulableRequest(this, buildRequest, parent);
request.CreationTime = EventTime;
LinkedListNode <SchedulableRequest> requestNode = _unscheduledRequests.AddFirst(request);
_unscheduledRequestNodesByRequest[request] = requestNode;
// Update the configuration information.
HashSet <SchedulableRequest> requests;
if (!_configurationToRequests.TryGetValue(request.BuildRequest.ConfigurationId, out requests))
{
requests = new HashSet <SchedulableRequest>();
_configurationToRequests[request.BuildRequest.ConfigurationId] = requests;
}
requests.Add(request);
// Update the build hierarchy.
if (!_buildHierarchy.ContainsKey(request))
{
_buildHierarchy[request] = new List <SchedulableRequest>(8);
}
if (parent != null)
{
ErrorUtilities.VerifyThrow(_buildHierarchy.ContainsKey(parent), "Parent doesn't exist in build hierarchy for request {0}", request.BuildRequest.GlobalRequestId);
_buildHierarchy[parent].Add(request);
}
return(request);
}
/// <summary>
/// Gets the name of the plan file for a specified submission.
/// </summary>
private string GetPlanName(SchedulableRequest rootRequest)
{
if (rootRequest == null)
{
return(null);
}
return(_configCache[rootRequest.BuildRequest.ConfigurationId].ProjectFullPath + ".buildplan");
}
/// <summary>
/// Explicitly grants CPU cores to a request.
/// </summary>
public void GrantCoresToRequest(int globalRequestId, int coresToGrant)
{
// Update per-request state.
SchedulableRequest request = GetScheduledRequest(globalRequestId);
request.GrantedCores += coresToGrant;
// Update global state.
_grantedCores += coresToGrant;
}
/// <summary>
/// Explicitly removes previously granted CPU cores from a request.
/// </summary>
public void RemoveCoresFromRequest(int globalRequestId, int coresToRemove)
{
// Update per-request state.
SchedulableRequest request = GetScheduledRequest(globalRequestId);
coresToRemove = Math.Min(request.GrantedCores, coresToRemove);
request.GrantedCores -= coresToRemove;
// Update global state.
_grantedCores -= coresToRemove;
}
/// <summary>
/// Reads a plan for the specified submission Id.
/// </summary>
public void ReadPlan(int submissionId, ILoggingService loggingService, BuildEventContext buildEventContext)
{
if (!BuildParameters.EnableBuildPlan)
{
return;
}
SchedulableRequest rootRequest = GetRootRequest(submissionId);
if (rootRequest == null)
{
return;
}
string planName = GetPlanName(rootRequest);
if (String.IsNullOrEmpty(planName))
{
return;
}
if (!File.Exists(planName))
{
return;
}
try
{
using (StreamReader file = new StreamReader(File.Open(planName, FileMode.Open)))
{
ReadTimes(file);
ReadHierarchy(file);
}
if (_configIdToData.Count > 0)
{
AnalyzeData();
}
}
catch (IOException)
{
loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("CantReadBuildPlan", planName));
}
catch (InvalidDataException)
{
loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("BuildPlanCorrupt", planName));
}
catch (FormatException)
{
loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("BuildPlanCorrupt", planName));
}
}
/// <summary>
/// Verifies that the request is scheduled and in the expected state.
/// </summary>
private void ExpectScheduledRequestState(int globalRequestId, SchedulableRequestState state)
{
SchedulableRequest request = InternalGetScheduledRequestByGlobalRequestId(globalRequestId);
if (request == null)
{
ErrorUtilities.ThrowInternalError("Request {0} was expected to be in state {1} but is not scheduled at all (it may be unscheduled or may be unknown to the system.)", globalRequestId, state);
}
else
{
request.VerifyState(state);
}
}
/// <summary>
/// Detects a circular dependency where the request which is about to block us is already blocked by us, usually as a result
/// of it having been previously scheduled in a multiproc scenario, but before this request was able to execute.
/// </summary>
/// <remarks>
/// Let A be 'this' project and B be 'blockingRequest' (the request which is going to block A.)
/// An indirect circular dependency exists if there is a dependency path from B to A. If there is no
/// existing blocked request B' with the same global request id as B, then there can be no path from B to A because B is a brand new
/// request with no other dependencies. If there is an existing blocked request B' with the same global request ID as B, then we
/// walk the set of dependencies recursively searching for A. If A is found, we have a circular dependency.
/// </remarks>
private void DetectIndirectCircularDependency(SchedulableRequest blockingRequest)
{
// If there is already a blocked request which has the same configuration id as the blocking request and that blocked request is (recursively)
// waiting on this request, then that is an indirect circular dependency.
SchedulableRequest alternateRequest = _schedulingData.GetBlockedRequestIfAny(blockingRequest.BuildRequest.GlobalRequestId);
if (alternateRequest == null)
{
return;
}
Stack <SchedulableRequest> requestsToEvaluate = new Stack <SchedulableRequest>(16);
HashSet <SchedulableRequest> evaluatedRequests = new HashSet <SchedulableRequest>();
requestsToEvaluate.Push(alternateRequest);
while (requestsToEvaluate.Count > 0)
{
SchedulableRequest requestToEvaluate = requestsToEvaluate.Pop();
// If we make it to a child which is us, then it's a circular dependency.
if (requestToEvaluate.BuildRequest.GlobalRequestId == this.BuildRequest.GlobalRequestId)
{
ThrowIndirectCircularDependency(blockingRequest, requestToEvaluate);
}
evaluatedRequests.Add(requestToEvaluate);
// If the request is not scheduled, it's possible that is because it's been scheduled elsewhere and is blocked.
// Follow that path if it exists.
if (requestToEvaluate.State == SchedulableRequestState.Unscheduled)
{
requestToEvaluate = _schedulingData.GetBlockedRequestIfAny(requestToEvaluate.BuildRequest.GlobalRequestId);
// If there was no scheduled request to evaluate, move on.
if (requestToEvaluate == null || evaluatedRequests.Contains(requestToEvaluate))
{
continue;
}
}
// This request didn't cause a circular dependency, check its children.
foreach (SchedulableRequest childRequest in requestToEvaluate.RequestsWeAreBlockedBy)
{
if (!evaluatedRequests.Contains(childRequest))
{
requestsToEvaluate.Push(childRequest);
}
}
}
}
/// <summary>
/// Recursively accumulates the amount of time spent in each configuration.
/// </summary>
private void RecursiveAccumulateConfigurationTimes(SchedulableRequest request, Dictionary <int, double> accumulatedTimeByConfiguration)
{
double accumulatedTime;
// NOTE: Do we want to count it each time the config appears in the hierarchy? This will inflate the
// cost of frequently referenced configurations.
accumulatedTimeByConfiguration.TryGetValue(request.BuildRequest.ConfigurationId, out accumulatedTime);
accumulatedTimeByConfiguration[request.BuildRequest.ConfigurationId] = accumulatedTime + request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalMilliseconds;
foreach (SchedulableRequest childRequest in _schedulingData.GetRequestsByHierarchy(request))
{
RecursiveAccumulateConfigurationTimes(childRequest, accumulatedTimeByConfiguration);
}
}
/// <summary>
/// Writes out all of the dependencies for a specified request, recursively.
/// </summary>
private void RecursiveWriteDependencies(StreamWriter file, SchedulableRequest request)
{
file.Write(request.BuildRequest.ConfigurationId);
foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request))
{
file.Write(" {0}", child.BuildRequest.ConfigurationId);
}
file.WriteLine();
foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request))
{
RecursiveWriteDependencies(file, child);
}
}
/// <summary>
/// Build our ancestor list then throw the circular dependency error.
/// </summary>
private void ThrowIndirectCircularDependency(SchedulableRequest blockingRequest, SchedulableRequest requestToEvaluate)
{
// We found a request which has the same global request ID as us in a chain which leads from the (already blocked) request
// which is trying to block us. Calculate its list of ancestors by walking up the parent list.
List <SchedulableRequest> ancestors = new List <SchedulableRequest>(16);
while (requestToEvaluate.Parent != null)
{
ancestors.Add(requestToEvaluate.Parent);
requestToEvaluate = requestToEvaluate.Parent;
}
ancestors.Reverse(); // Because the list should be in the order from root to child.
CleanupForCircularDependencyAndThrow(blockingRequest, ancestors);
}
/// <summary>
/// Writes a plan for the specified submission id.
/// </summary>
public void WritePlan(int submissionId, ILoggingService loggingService, BuildEventContext buildEventContext)
{
if (!BuildParameters.EnableBuildPlan)
{
return;
}
SchedulableRequest rootRequest = GetRootRequest(submissionId);
if (rootRequest == null)
{
return;
}
string planName = GetPlanName(rootRequest);
if (String.IsNullOrEmpty(planName))
{
return;
}
try
{
using (StreamWriter file = new StreamWriter(File.Open(planName, FileMode.Create)))
{
// Write the accumulated configuration times.
Dictionary <int, double> accumulatedTimeByConfiguration = new Dictionary <int, double>();
RecursiveAccumulateConfigurationTimes(rootRequest, accumulatedTimeByConfiguration);
List <int> configurationsInOrder = new List <int>(accumulatedTimeByConfiguration.Keys);
configurationsInOrder.Sort();
foreach (int configId in configurationsInOrder)
{
file.WriteLine(String.Format(CultureInfo.InvariantCulture, "{0} {1} {2}", configId, accumulatedTimeByConfiguration[configId], _configCache[configId].ProjectFullPath));
}
file.WriteLine();
// Write out the dependency information.
RecursiveWriteDependencies(file, rootRequest);
}
}
catch (IOException)
{
loggingService.LogCommentFromText(buildEventContext, MessageImportance.Low, ResourceUtilities.FormatResourceString("CantWriteBuildPlan", planName));
}
}
/// <summary>
/// Retrieves a set of build requests which have the specified parent. If root is null, this will retrieve all of the
/// top-level requests.
/// </summary>
public IEnumerable <SchedulableRequest> GetRequestsByHierarchy(SchedulableRequest root)
{
if (root == null)
{
// Retrieve all requests which are roots of the tree.
List <SchedulableRequest> roots = new List <SchedulableRequest>();
foreach (SchedulableRequest key in _buildHierarchy.Keys)
{
if (key.Parent == null)
{
roots.Add(key);
}
}
return(roots);
}
return(_buildHierarchy[root]);
}
/// <summary>
/// Detects a circular dependency where the blocking request is in our direct ancestor chain.
/// </summary>
private void DetectDirectCircularDependency(SchedulableRequest blockingRequest)
{
// A circular dependency occurs when this project (or any of its ancestors) has the same global request id as the
// blocking request.
List <SchedulableRequest> ancestors = new List <SchedulableRequest>(16);
SchedulableRequest currentRequest = this;
do
{
ancestors.Add(currentRequest);
if (currentRequest.BuildRequest.GlobalRequestId == blockingRequest.BuildRequest.GlobalRequestId)
{
// We are directly conflicting with an instance of ourselves.
CleanupForCircularDependencyAndThrow(blockingRequest, ancestors);
}
currentRequest = currentRequest.Parent;
}while (currentRequest != null);
}
/// <summary>
/// Removes the association between this request and the one we are blocked by.
/// </summary>
private void DisconnectRequestWeAreBlockedBy(BlockingRequestKey blockingRequestKey)
{
ErrorUtilities.VerifyThrow(_requestsWeAreBlockedBy.ContainsKey(blockingRequestKey), "We are not blocked by the specified request.");
SchedulableRequest unblockingRequest = _requestsWeAreBlockedBy[blockingRequestKey];
ErrorUtilities.VerifyThrow(unblockingRequest._requestsWeAreBlocking.Contains(this), "The request unblocking us doesn't think it is blocking us.");
_requestsWeAreBlockedBy.Remove(blockingRequestKey);
unblockingRequest._requestsWeAreBlocking.Remove(this);
// If the request we are blocked by also happens to be unscheduled, remove it as well so we don't try to run it later. This is
// because circular dependency errors cause us to fail all outstanding requests on the current request. See BuildRequsetEntry.ReportResult.
if (unblockingRequest.State == SchedulableRequestState.Unscheduled)
{
unblockingRequest.Delete();
}
if (_requestsWeAreBlockedBy.Count == 0)
{
ChangeToState(SchedulableRequestState.Ready);
}
}
/// <summary>
/// Constructor.
/// </summary>
public SchedulableRequest(SchedulingData collection, BuildRequest request, SchedulableRequest parent)
{
ErrorUtilities.VerifyThrowArgumentNull(collection, "collection");
ErrorUtilities.VerifyThrowArgumentNull(request, "request");
ErrorUtilities.VerifyThrow((parent == null) || (parent._schedulingData == collection), "Parent request does not belong to the same collection.");
_schedulingData = collection;
_request = request;
_parent = parent;
_assignedNodeId = -1;
_requestsWeAreBlockedBy = new Dictionary <BlockingRequestKey, SchedulableRequest>();
_requestsWeAreBlocking = new HashSet <SchedulableRequest>();
_timeRecords = new Dictionary <SchedulableRequestState, ScheduleTimeRecord>(5);
_timeRecords[SchedulableRequestState.Unscheduled] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Blocked] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Yielding] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Executing] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Ready] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Completed] = new ScheduleTimeRecord();
ChangeToState(SchedulableRequestState.Unscheduled);
}
/// <summary>
/// Determines if the specified request is currently scheduled.
/// </summary>
public bool IsRequestScheduled(SchedulableRequest request)
{
return InternalGetScheduledRequestByGlobalRequestId(request.BuildRequest.GlobalRequestId) != null;
}
/// <summary>
/// Marks the parent as blocked waiting for results from a results transfer.
/// </summary>
private void HandleRequestBlockedOnResultsTransfer(SchedulableRequest parentRequest, List<ScheduleResponse> responses)
{
// Create the new request which will go to the configuration's results node.
BuildRequest newRequest = new BuildRequest(parentRequest.BuildRequest.SubmissionId, BuildRequest.ResultsTransferNodeRequestId, parentRequest.BuildRequest.ConfigurationId, new string[0], null, parentRequest.BuildRequest.BuildEventContext, parentRequest.BuildRequest, parentRequest.BuildRequest.BuildRequestDataFlags);
// Assign a new global request id - always different from any other.
newRequest.GlobalRequestId = _nextGlobalRequestId;
_nextGlobalRequestId++;
// Now add the response. Send it to the node where the configuration's results are stored. When those results come back
// we will update the storage location in the configuration. This is doing a bit of a run around the scheduler - we don't
// create a new formal request, so we treat the blocked request as if it is still executing - this prevents any other requests
// from getting onto that node and also means we don't have to do additional work to get the scheduler to understand the bizarre
// case of sending a request for results from a project's own configuration (which it believes reside on the very node which
// is actually requesting the results in the first place.)
BuildRequestConfiguration configuration = _configCache[parentRequest.BuildRequest.ConfigurationId];
responses.Add(ScheduleResponse.CreateScheduleResponse(configuration.ResultsNodeId, newRequest, false));
TraceScheduler("Created request {0} (node request {1}) for transfer of configuration {2}'s results from node {3} to node {4}", newRequest.GlobalRequestId, newRequest.NodeRequestId, configuration.ConfigurationId, configuration.ResultsNodeId, parentRequest.AssignedNode);
// The configuration's results will now be homed at the new location (once they have come back from the
// original node.)
configuration.ResultsNodeId = parentRequest.AssignedNode;
}
/// <summary>
/// Returns true if the request can be scheduled to the specified node.
/// </summary>
public bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId)
{
int requiredNodeId = GetAssignedNodeForRequestConfiguration(request.BuildRequest.ConfigurationId);
return(requiredNodeId == Scheduler.InvalidNodeId || requiredNodeId == nodeId);
}
/// <summary>
/// Dumps the hierarchy of requests.
/// </summary>
private void DumpRequestHierarchy(StreamWriter file, SchedulableRequest root, int indent)
{
foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(root))
{
DumpRequestSpec(file, child, indent, null);
DumpRequestHierarchy(file, child, indent + 1);
}
}
/// <summary>
/// Creates a new request and adds it to the system
/// </summary>
/// <remarks>
/// New requests always go on the front of the queue, because we prefer to build the projects we just received first (depth first, absent
/// any particular scheduling algorithm such as in the single-proc case.)
/// </remarks>
public SchedulableRequest CreateRequest(BuildRequest buildRequest, SchedulableRequest parent)
{
SchedulableRequest request = new SchedulableRequest(this, buildRequest, parent);
request.CreationTime = EventTime;
LinkedListNode<SchedulableRequest> requestNode = _unscheduledRequests.AddFirst(request);
_unscheduledRequestNodesByRequest[request] = requestNode;
// Update the configuration information.
HashSet<SchedulableRequest> requests;
if (!_configurationToRequests.TryGetValue(request.BuildRequest.ConfigurationId, out requests))
{
requests = new HashSet<SchedulableRequest>();
_configurationToRequests[request.BuildRequest.ConfigurationId] = requests;
}
requests.Add(request);
// Update the build hierarchy.
if (!_buildHierarchy.ContainsKey(request))
{
_buildHierarchy[request] = new List<SchedulableRequest>(8);
}
if (parent != null)
{
ErrorUtilities.VerifyThrow(_buildHierarchy.ContainsKey(parent), "Parent doesn't exist in build hierarchy for request {0}", request.BuildRequest.GlobalRequestId);
_buildHierarchy[parent].Add(request);
}
return request;
}
/// <summary>
/// Marks the specified request and all of its ancestors as having aborted.
/// </summary>
private void MarkRequestAborted(SchedulableRequest request)
{
_resultsCache.AddResult(new BuildResult(request.BuildRequest, new BuildAbortedException()));
// Recursively abort all of the requests we are blocking.
foreach (SchedulableRequest blockedRequest in request.RequestsWeAreBlocking)
{
MarkRequestAborted(blockedRequest);
}
}
/// <summary>
/// Detects a circular dependency. Throws a CircularDependencyException if one exists. Circular dependencies can occur
/// under the following conditions:
/// 1. If the blocking request's global request ID appears in the ancestor chain (Direct).
/// 2. If a request appears in the ancestor chain and has a different global request ID but has an active target that
/// matches one of the targets specified in the blocking request (Direct).
/// 3. If the blocking request exists elsewhere as a blocked request with the same global request ID, and one of its children
/// (recursively) matches this request's global request ID (Indirect).
/// 4. If the blocking request's configuration is part of another request elsewhere which is also blocked, and that request
/// is building targets this blocking request is building, and one of that blocked request's children (recursively)
/// matches this request's global request ID (Indirect).
/// </summary>
private void DetectCircularDependency(SchedulableRequest blockingRequest)
{
DetectDirectCircularDependency(blockingRequest);
DetectIndirectCircularDependency(blockingRequest);
}
/// <summary>
/// This method determines how many requests are waiting for this request, taking into account the full tree of all requests
/// in all dependency chains which are waiting.
/// </summary>
private int ComputeClosureOfWaitingRequests(SchedulableRequest request)
{
int waitingRequests = 0;
// In single-proc, this doesn't matter since scheduling is always 100% efficient.
if (_componentHost.BuildParameters.MaxNodeCount > 1)
{
foreach (SchedulableRequest waitingRequest in request.RequestsWeAreBlocking)
{
waitingRequests++;
waitingRequests += ComputeClosureOfWaitingRequests(waitingRequest);
}
}
return waitingRequests;
}
/// <summary>
/// Iterates through the set of available nodes and checks whether any of them is
/// capable of servicing this request or any of the requests that it is blocked
/// by (regardless of whether they are currently available to do so).
/// </summary>
private bool RequestOrAnyItIsBlockedByCanBeServiced(SchedulableRequest request)
{
if (request.RequestsWeAreBlockedByCount > 0)
{
foreach (SchedulableRequest requestWeAreBlockedBy in request.RequestsWeAreBlockedBy)
{
if (RequestOrAnyItIsBlockedByCanBeServiced(requestWeAreBlockedBy))
{
return true;
}
}
// if none of the requests we are blocked by can be serviced, it doesn't matter
// whether we can be serviced or not -- the reason we're blocked is because none
// of the requests we are blocked by can be serviced.
return false;
}
else
{
foreach (NodeInfo node in _availableNodes.Values)
{
if (CanScheduleRequestToNode(request, node.NodeId))
{
return true;
}
}
return false;
}
}
/// <summary>
/// Returns true if the request can be scheduled to the specified node.
/// </summary>
public bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId)
{
int requiredNodeId = GetAssignedNodeForRequestConfiguration(request.BuildRequest.ConfigurationId);
return requiredNodeId == Scheduler.InvalidNodeId || requiredNodeId == nodeId;
}
/// <summary>
/// Returns true if the request can be scheduled to the specified node.
/// </summary>
public bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId)
{
return(CanScheduleConfigurationToNode(request.BuildRequest.ConfigurationId, nodeId));
}
/// <summary>
/// Updates the state of a request based on its desire to yield or reacquire control of its node.
/// </summary>
private void HandleYieldAction(SchedulableRequest parentRequest, BuildRequestBlocker blocker)
{
if (blocker.YieldAction == YieldAction.Yield)
{
// Mark the request blocked.
parentRequest.Yield(blocker.TargetsInProgress);
}
else
{
// Mark the request ready.
parentRequest.Reacquire();
}
}
/// <summary>
/// Dumps detailed information about a request.
/// </summary>
private void DumpRequestSpec(StreamWriter file, SchedulableRequest request, int indent, string prefix)
{
file.WriteLine("{0}{1}{2}: [{3}] {4}{5} ({6}){7} ({8})", new String(' ', indent * 2), (prefix == null) ? "" : prefix, request.BuildRequest.GlobalRequestId, _schedulingData.GetAssignedNodeForRequestConfiguration(request.BuildRequest.ConfigurationId), _schedulingData.IsRequestScheduled(request) ? "RUNNING " : "", request.State, request.BuildRequest.ConfigurationId, _configCache[request.BuildRequest.ConfigurationId].ProjectFullPath, String.Join(", ", request.BuildRequest.Targets.ToArray()));
}
/// <summary>
/// Dumps the state of a request.
/// </summary>
private void DumpRequestState(StreamWriter file, SchedulableRequest request, int indent)
{
DumpRequestSpec(file, request, indent, null);
if (request.RequestsWeAreBlockedByCount > 0)
{
foreach (SchedulableRequest blockingRequest in request.RequestsWeAreBlockedBy)
{
DumpRequestSpec(file, blockingRequest, indent + 1, "!");
}
}
if (request.RequestsWeAreBlockingCount > 0)
{
foreach (SchedulableRequest blockedRequest in request.RequestsWeAreBlocking)
{
DumpRequestSpec(file, blockedRequest, indent + 1, ">");
}
}
}
/// <summary>
/// Attempts to get results from the cache for this request. If results are available, reports them to the
/// correct node. If that action causes the parent to become ready and its node is idle, the parent is
/// resumed.
/// </summary>
private void ResolveRequestFromCacheAndResumeIfPossible(SchedulableRequest request, List<ScheduleResponse> responses)
{
int nodeForResults = (request.Parent != null) ? request.Parent.AssignedNode : InvalidNodeId;
// Do we already have results? If so, just return them.
ScheduleResponse response = TrySatisfyRequestFromCache(nodeForResults, request.BuildRequest, skippedResultsAreOK: false);
if (response != null)
{
if (response.Action == ScheduleActionType.SubmissionComplete)
{
ErrorUtilities.VerifyThrow(request.Parent == null, "Unexpectedly generated a SubmissionComplete response for a request which is not top-level.");
LogRequestHandledFromCache(request.BuildRequest, response.BuildResult);
// This was root request, we can report submission complete.
responses.Add(ScheduleResponse.CreateSubmissionCompleteResponse(response.BuildResult));
if (response.BuildResult.OverallResult != BuildResultCode.Failure)
{
WriteSchedulingPlan(response.BuildResult.SubmissionId);
}
}
else
{
LogRequestHandledFromCache(request.BuildRequest, response.Unblocker.Result);
request.Complete(response.Unblocker.Result);
TraceScheduler("Reporting results for request {0} with parent {1} to node {2} from cache.", request.BuildRequest.GlobalRequestId, request.BuildRequest.ParentGlobalRequestId, response.NodeId);
if (response.NodeId != InvalidNodeId)
{
responses.Add(response);
}
// Is the node we are reporting to idle? If so, does reporting this result allow it to proceed with work?
if (!_schedulingData.IsNodeWorking(response.NodeId))
{
ResumeReadyRequestIfAny(response.NodeId, responses);
}
}
}
}
/// <summary>
/// Updates the state of the specified request.
/// </summary>
public void UpdateFromState(SchedulableRequest request, SchedulableRequestState previousState)
{
// Remove from its old collection
switch (previousState)
{
case SchedulableRequestState.Blocked:
_blockedRequests.Remove(request.BuildRequest.GlobalRequestId);
break;
case SchedulableRequestState.Yielding:
_yieldingRequests.Remove(request.BuildRequest.GlobalRequestId);
break;
case SchedulableRequestState.Completed:
ErrorUtilities.ThrowInternalError("Should not be updating a request after it has reached the Completed state.");
break;
case SchedulableRequestState.Executing:
_executingRequests.Remove(request.BuildRequest.GlobalRequestId);
_executingRequestByNode[request.AssignedNode] = null;
break;
case SchedulableRequestState.Ready:
_readyRequests.Remove(request.BuildRequest.GlobalRequestId);
_readyRequestsByNode[request.AssignedNode].Remove(request);
break;
case SchedulableRequestState.Unscheduled:
LinkedListNode<SchedulableRequest> requestNode = _unscheduledRequestNodesByRequest[request];
_unscheduledRequestNodesByRequest.Remove(request);
_unscheduledRequests.Remove(requestNode);
if (request.State != SchedulableRequestState.Completed)
{
// Map the request to the node.
HashSet<SchedulableRequest> requestsAssignedToNode;
if (!_scheduledRequestsByNode.TryGetValue(request.AssignedNode, out requestsAssignedToNode))
{
requestsAssignedToNode = new HashSet<SchedulableRequest>();
_scheduledRequestsByNode[request.AssignedNode] = requestsAssignedToNode;
}
ErrorUtilities.VerifyThrow(!requestsAssignedToNode.Contains(request), "Request {0} is already scheduled to node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode);
requestsAssignedToNode.Add(request);
// Map the configuration to the node.
HashSet<int> configurationsAssignedToNode;
if (!_configurationsByNode.TryGetValue(request.AssignedNode, out configurationsAssignedToNode))
{
configurationsAssignedToNode = new HashSet<int>();
_configurationsByNode[request.AssignedNode] = configurationsAssignedToNode;
}
if (!configurationsAssignedToNode.Contains(request.BuildRequest.ConfigurationId))
{
configurationsAssignedToNode.Add(request.BuildRequest.ConfigurationId);
}
}
break;
}
// Add it to its new location
switch (request.State)
{
case SchedulableRequestState.Blocked:
ErrorUtilities.VerifyThrow(!_blockedRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already blocked!");
_blockedRequests[request.BuildRequest.GlobalRequestId] = request;
break;
case SchedulableRequestState.Yielding:
ErrorUtilities.VerifyThrow(!_yieldingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already yielded!");
_yieldingRequests[request.BuildRequest.GlobalRequestId] = request;
break;
case SchedulableRequestState.Completed:
ErrorUtilities.VerifyThrow(_configurationToRequests.ContainsKey(request.BuildRequest.ConfigurationId), "Configuration {0} never had requests assigned to it.", request.BuildRequest.ConfigurationId);
ErrorUtilities.VerifyThrow(_configurationToRequests[request.BuildRequest.ConfigurationId].Count > 0, "Configuration {0} has no requests assigned to it.", request.BuildRequest.ConfigurationId);
_configurationToRequests[request.BuildRequest.ConfigurationId].Remove(request);
if (_scheduledRequestsByNode.ContainsKey(request.AssignedNode))
{
_scheduledRequestsByNode[request.AssignedNode].Remove(request);
}
request.EndTime = EventTime;
break;
case SchedulableRequestState.Executing:
ErrorUtilities.VerifyThrow(!_executingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already executing!");
ErrorUtilities.VerifyThrow(!_executingRequestByNode.ContainsKey(request.AssignedNode) || _executingRequestByNode[request.AssignedNode] == null, "Node {0} is currently executing a request.", request.AssignedNode);
_executingRequests[request.BuildRequest.GlobalRequestId] = request;
_executingRequestByNode[request.AssignedNode] = request;
_configurationToNode[request.BuildRequest.ConfigurationId] = request.AssignedNode;
if (previousState == SchedulableRequestState.Unscheduled)
{
request.StartTime = EventTime;
}
break;
case SchedulableRequestState.Ready:
ErrorUtilities.VerifyThrow(!_readyRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already ready!");
_readyRequests[request.BuildRequest.GlobalRequestId] = request;
HashSet<SchedulableRequest> readyRequestsOnNode;
if (!_readyRequestsByNode.TryGetValue(request.AssignedNode, out readyRequestsOnNode))
{
readyRequestsOnNode = new HashSet<SchedulableRequest>();
_readyRequestsByNode[request.AssignedNode] = readyRequestsOnNode;
}
ErrorUtilities.VerifyThrow(!readyRequestsOnNode.Contains(request), "Request with global id {0} is already marked as ready on node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode);
readyRequestsOnNode.Add(request);
break;
case SchedulableRequestState.Unscheduled:
ErrorUtilities.ThrowInternalError("Request with global id {0} cannot transition to the Unscheduled state", request.BuildRequest.GlobalRequestId);
break;
}
_buildEvents.Add(new SchedulingEvent(EventTime, request, previousState, request.State));
}
/// <summary>
/// Marks this request as being blocked by the specified request. Establishes the correct relationships between the requests.
/// </summary>
/// <param name="blockingRequest">The request which is blocking this one.</param>
/// <param name="activeTargets">The list of targets this request was currently building at the time it became blocked.</param>
public void BlockByRequest(SchedulableRequest blockingRequest, string[] activeTargets)
{
VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Blocked, SchedulableRequestState.Executing });
ErrorUtilities.VerifyThrowArgumentNull(blockingRequest, "blockingRequest");
ErrorUtilities.VerifyThrowArgumentNull(activeTargets, "activeTargets");
// Note that the blocking request will typically be our parent UNLESS it is a request we blocked on because it was executing a target we wanted to execute.
// Thus, we do not assert the parent-child relationship here.
BlockingRequestKey key = new BlockingRequestKey(blockingRequest.BuildRequest);
ErrorUtilities.VerifyThrow(!_requestsWeAreBlockedBy.ContainsKey(key), "We are already blocked by this request.");
ErrorUtilities.VerifyThrow(!blockingRequest._requestsWeAreBlocking.Contains(this), "The blocking request thinks it is already blocking us.");
// This method is only called when a request reports that it is blocked on other requests. If the request is being blocked by a brand new
// request, that request will be unscheduled. If this request is blocked by an in-progress request which was executing a target it needed
// to also execute, then that request is not unscheduled (because it was running on the node) and it is not executing (because this condition
// can only occur against requests which are executing on the same node and since the request which called this method is the one currently
// executing on that node, that means the request it is blocked by must either be itself blocked or ready.)
blockingRequest.VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Yielding, SchedulableRequestState.Blocked, SchedulableRequestState.Ready, SchedulableRequestState.Unscheduled });
// Update our list of active targets. This has to be done before we detect circular dependencies because we use this information to detect
// re-entrancy circular dependencies.
_activeTargetsWhenBlocked = activeTargets;
DetectCircularDependency(blockingRequest);
_requestsWeAreBlockedBy[key] = blockingRequest;
blockingRequest._requestsWeAreBlocking.Add(this);
ChangeToState(SchedulableRequestState.Blocked);
}
/// <summary>
/// Marks the request as being blocked by another request which is currently building a target whose results we need to proceed.
/// </summary>
private void HandleRequestBlockedOnInProgressTarget(SchedulableRequest blockedRequest, BuildRequestBlocker blocker)
{
ErrorUtilities.VerifyThrowArgumentNull(blockedRequest, "blockedRequest");
ErrorUtilities.VerifyThrowArgumentNull(blocker, "blocker");
// We are blocked on an in-progress request building a target whose results we need.
SchedulableRequest blockingRequest = _schedulingData.GetScheduledRequest(blocker.BlockingRequestId);
// The request we blocked on couldn't have been executing (because we are) so it must either be yielding (which is ok because
// it isn't modifying its own state, just running a background process), ready, or still blocked.
blockingRequest.VerifyOneOfStates(new SchedulableRequestState[] { SchedulableRequestState.Yielding, SchedulableRequestState.Ready, SchedulableRequestState.Blocked });
blockedRequest.BlockByRequest(blockingRequest, blocker.TargetsInProgress);
}
/// <summary>
/// Assigns the specified request to the specified node.
/// </summary>
private void AssignUnscheduledRequestToNode(SchedulableRequest request, int nodeId, List<ScheduleResponse> responses)
{
ErrorUtilities.VerifyThrowArgumentNull(request, "request");
ErrorUtilities.VerifyThrowArgumentNull(responses, "responses");
ErrorUtilities.VerifyThrow(nodeId != InvalidNodeId, "Invalid node id specified.");
// Currently we cannot move certain kinds of traversals (notably solution metaprojects) to other nodes because
// they only have a ProjectInstance representation, and besides these kinds of projects build very quickly
// and produce more references (more work to do.) This just verifies we do not attempt to send a traversal to
// an out-of-proc node because doing so is inefficient and presently will cause the engine to fail on the remote
// node because these projects cannot be found.
ErrorUtilities.VerifyThrow(nodeId == InProcNodeId || _forceAffinityOutOfProc || !IsTraversalRequest(request.BuildRequest), "Can't assign traversal request to out-of-proc node!");
request.VerifyState(SchedulableRequestState.Unscheduled);
// Determine if this node has seen our configuration before. If not, we must send it along with this request.
bool mustSendConfigurationToNode = _availableNodes[nodeId].AssignConfiguration(request.BuildRequest.ConfigurationId);
// If this is the first time this configuration has been assigned to a node, we will mark the configuration with the assigned node
// indicating that the master set of results is located there. Should we ever need to move the results, we will know where to find them.
BuildRequestConfiguration config = _configCache[request.BuildRequest.ConfigurationId];
if (config.ResultsNodeId == InvalidNodeId)
{
config.ResultsNodeId = nodeId;
}
ErrorUtilities.VerifyThrow(config.ResultsNodeId != InvalidNodeId, "Configuration's results node is not set.");
responses.Add(ScheduleResponse.CreateScheduleResponse(nodeId, request.BuildRequest, mustSendConfigurationToNode));
TraceScheduler("Executing request {0} on node {1} with parent {2}", request.BuildRequest.GlobalRequestId, nodeId, (request.Parent == null) ? -1 : request.Parent.BuildRequest.GlobalRequestId);
request.ResumeExecution(nodeId);
}
/// <summary>
/// Returns true if a request can be scheduled to a node, false otherwise.
/// </summary>
private bool CanScheduleRequestToNode(SchedulableRequest request, int nodeId)
{
if (_schedulingData.CanScheduleRequestToNode(request, nodeId))
{
NodeAffinity affinity = GetNodeAffinityForRequest(request.BuildRequest);
bool result = _availableNodes[nodeId].CanServiceRequestWithAffinity(affinity);
return result;
}
return false;
}
/// <summary>
/// Marks the request as being blocked by new requests whose results we must get before we can proceed.
/// </summary>
private void HandleRequestBlockedByNewRequests(SchedulableRequest parentRequest, BuildRequestBlocker blocker, List<ScheduleResponse> responses)
{
ErrorUtilities.VerifyThrowArgumentNull(blocker, "blocker");
ErrorUtilities.VerifyThrowArgumentNull(responses, "responses");
// The request is waiting on new requests.
bool abortRequestBatch = false;
Stack<BuildRequest> requestsToAdd = new Stack<BuildRequest>(blocker.BuildRequests.Length);
foreach (BuildRequest request in blocker.BuildRequests)
{
// Assign a global request id to this request.
if (request.GlobalRequestId == BuildRequest.InvalidGlobalRequestId)
{
AssignGlobalRequestId(request);
}
int nodeForResults = (parentRequest == null) ? InvalidNodeId : parentRequest.AssignedNode;
TraceScheduler("Received request {0} (node request {1}) with parent {2} from node {3}", request.GlobalRequestId, request.NodeRequestId, request.ParentGlobalRequestId, nodeForResults);
// First, determine if we have already built this request and have results for it. If we do, we prepare the responses for it
// directly here. We COULD simply report these as blocking the parent request and let the scheduler pick them up later when the parent
// comes back up as schedulable, but we prefer to send the results back immediately so this request can (potentially) continue uninterrupted.
ScheduleResponse response = TrySatisfyRequestFromCache(nodeForResults, request, skippedResultsAreOK: false);
if (null != response)
{
TraceScheduler("Request {0} (node request {1}) satisfied from the cache.", request.GlobalRequestId, request.NodeRequestId);
// BuildResult result = (response.Action == ScheduleActionType.Unblock) ? response.Unblocker.Results[0] : response.BuildResult;
LogRequestHandledFromCache(request, response.BuildResult);
responses.Add(response);
// If we wish to implement an algorithm where the first failing request aborts the remaining request, check for
// overall result being failure rather than just circular dependency. Sync with BasicScheduler.ReportResult and
// BuildRequestEntry.ReportResult.
if (response.BuildResult.CircularDependency)
{
abortRequestBatch = true;
}
}
else
{
// Ensure there is no affinity mismatch between this request and a previous request of the same configuration.
NodeAffinity requestAffinity = GetNodeAffinityForRequest(request);
NodeAffinity existingRequestAffinity = NodeAffinity.Any;
if (requestAffinity != NodeAffinity.Any)
{
bool affinityMismatch = false;
int assignedNodeId = _schedulingData.GetAssignedNodeForRequestConfiguration(request.ConfigurationId);
if (assignedNodeId != Scheduler.InvalidNodeId)
{
if (!_availableNodes[assignedNodeId].CanServiceRequestWithAffinity(GetNodeAffinityForRequest(request)))
{
// This request's configuration has already been assigned to a node which cannot service this affinity.
if (_schedulingData.GetRequestsAssignedToConfigurationCount(request.ConfigurationId) == 0)
{
// If there are no other requests already scheduled for that configuration, we can safely reassign.
_schedulingData.UnassignNodeForRequestConfiguration(request.ConfigurationId);
}
else
{
existingRequestAffinity = (_availableNodes[assignedNodeId].ProviderType == NodeProviderType.InProc) ? NodeAffinity.InProc : NodeAffinity.OutOfProc;
affinityMismatch = true;
}
}
}
else if (_schedulingData.GetRequestsAssignedToConfigurationCount(request.ConfigurationId) > 0)
{
// Would any other existing requests for this configuration mismatch?
foreach (SchedulableRequest existingRequest in _schedulingData.GetRequestsAssignedToConfiguration(request.ConfigurationId))
{
existingRequestAffinity = GetNodeAffinityForRequest(existingRequest.BuildRequest);
if (existingRequestAffinity != NodeAffinity.Any && existingRequestAffinity != requestAffinity)
{
// The existing request has an affinity which doesn't match this one, so this one could never be scheduled.
affinityMismatch = true;
break;
}
}
}
if (affinityMismatch)
{
BuildResult result = new BuildResult(request, new InvalidOperationException(ResourceUtilities.FormatResourceString("AffinityConflict", requestAffinity, existingRequestAffinity)));
response = GetResponseForResult(nodeForResults, request, result);
responses.Add(response);
continue;
}
}
// Now add the requests so they would naturally be picked up by the scheduler in the order they were issued,
// but before any other requests in the list. This makes us prefer a depth-first traversal.
requestsToAdd.Push(request);
}
}
// Now add any unassigned build requests.
if (!abortRequestBatch)
{
if (requestsToAdd.Count == 0)
{
// All of the results are being reported directly from the cache (from above), so this request can continue on its merry way.
if (parentRequest != null)
{
// responses.Add(new ScheduleResponse(parentRequest.AssignedNode, new BuildRequestUnblocker(parentRequest.BuildRequest.GlobalRequestId)));
responses.Add(ScheduleResponse.CreateResumeExecutionResponse(parentRequest.AssignedNode, parentRequest.BuildRequest.GlobalRequestId));
}
}
else
{
while (requestsToAdd.Count > 0)
{
BuildRequest requestToAdd = requestsToAdd.Pop();
SchedulableRequest blockingRequest = _schedulingData.CreateRequest(requestToAdd, parentRequest);
if (parentRequest != null)
{
parentRequest.BlockByRequest(blockingRequest, blocker.TargetsInProgress);
}
}
}
}
}
/// <summary>
/// Detects a circular dependency where the request which is about to block us is already blocked by us, usually as a result
/// of it having been previously scheduled in a multiproc scenario, but before this request was able to execute.
/// </summary>
/// <remarks>
/// Let A be 'this' project and B be 'blockingRequest' (the request which is going to block A.)
/// An indirect circular dependency exists if there is a dependency path from B to A. If there is no
/// existing blocked request B' with the same global request id as B, then there can be no path from B to A because B is a brand new
/// request with no other dependencies. If there is an existing blocked request B' with the same global request ID as B, then we
/// walk the set of dependencies recursively searching for A. If A is found, we have a circular dependency.
/// </remarks>
private void DetectIndirectCircularDependency(SchedulableRequest blockingRequest)
{
// If there is already a blocked request which has the same configuration id as the blocking request and that blocked request is (recursively)
// waiting on this request, then that is an indirect circular dependency.
SchedulableRequest alternateRequest = _schedulingData.GetBlockedRequestIfAny(blockingRequest.BuildRequest.GlobalRequestId);
if (alternateRequest == null)
{
return;
}
Stack<SchedulableRequest> requestsToEvaluate = new Stack<SchedulableRequest>(16);
HashSet<SchedulableRequest> evaluatedRequests = new HashSet<SchedulableRequest>();
requestsToEvaluate.Push(alternateRequest);
while (requestsToEvaluate.Count > 0)
{
SchedulableRequest requestToEvaluate = requestsToEvaluate.Pop();
// If we make it to a child which is us, then it's a circular dependency.
if (requestToEvaluate.BuildRequest.GlobalRequestId == this.BuildRequest.GlobalRequestId)
{
ThrowIndirectCircularDependency(blockingRequest, requestToEvaluate);
}
evaluatedRequests.Add(requestToEvaluate);
// If the request is not scheduled, it's possible that is because it's been scheduled elsewhere and is blocked.
// Follow that path if it exists.
if (requestToEvaluate.State == SchedulableRequestState.Unscheduled)
{
requestToEvaluate = _schedulingData.GetBlockedRequestIfAny(requestToEvaluate.BuildRequest.GlobalRequestId);
// If there was no scheduled request to evaluate, move on.
if (requestToEvaluate == null || evaluatedRequests.Contains(requestToEvaluate))
{
continue;
}
}
// This request didn't cause a circular dependency, check its children.
foreach (SchedulableRequest childRequest in requestToEvaluate.RequestsWeAreBlockedBy)
{
if (!evaluatedRequests.Contains(childRequest))
{
requestsToEvaluate.Push(childRequest);
}
}
}
}
/// <summary>
/// Recursively dumps the build information for the specified hierarchy
/// </summary>
private void WriteRecursiveSummary(ILoggingService loggingService, BuildEventContext context, int submissionId, SchedulableRequest request, int level, bool useConfigurations, bool isLastChild)
{
int postPad = Math.Max(20 /* field width */ - (2 * level) /* spacing for hierarchy lines */ - 3 /* length allocated for config/request id */, 0);
StringBuilder prePadString = new StringBuilder(2 * level);
if (level != 0)
{
int levelsToPad = level;
if (isLastChild)
{
levelsToPad--;
}
while (levelsToPad > 0)
{
prePadString.Append("| ");
levelsToPad--;
}
if (isLastChild)
{
prePadString.Append(@". ");
}
}
loggingService.LogComment
(
context,
MessageImportance.Normal,
"BuildHierarchyEntry",
prePadString.ToString(),
useConfigurations ? request.BuildRequest.ConfigurationId : request.BuildRequest.GlobalRequestId,
new String(' ', postPad),
String.Format(CultureInfo.InvariantCulture, "{0:0.000}", request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalSeconds),
String.Format(CultureInfo.InvariantCulture, "{0:0.000}", request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalSeconds + request.GetTimeSpentInState(SchedulableRequestState.Blocked).TotalSeconds + request.GetTimeSpentInState(SchedulableRequestState.Ready).TotalSeconds),
_configCache[request.BuildRequest.ConfigurationId].ProjectFullPath,
String.Join(", ", request.BuildRequest.Targets.ToArray())
);
List<SchedulableRequest> childRequests = new List<SchedulableRequest>(_schedulingData.GetRequestsByHierarchy(request));
childRequests.Sort(delegate (SchedulableRequest left, SchedulableRequest right)
{
if (left.StartTime < right.StartTime)
{
return -1;
}
else if (left.StartTime > right.StartTime)
{
return 1;
}
return 0;
});
for (int i = 0; i < childRequests.Count; i++)
{
SchedulableRequest childRequest = childRequests[i];
WriteRecursiveSummary(loggingService, context, submissionId, childRequest, level + 1, useConfigurations, i == childRequests.Count - 1);
}
}
/// <summary>
/// Recursively accumulates the amount of time spent in each configuration.
/// </summary>
private void RecursiveAccumulateConfigurationTimes(SchedulableRequest request, Dictionary<int, double> accumulatedTimeByConfiguration)
{
double accumulatedTime;
// NOTE: Do we want to count it each time the config appears in the hierarchy? This will inflate the
// cost of frequently referenced configurations.
accumulatedTimeByConfiguration.TryGetValue(request.BuildRequest.ConfigurationId, out accumulatedTime);
accumulatedTimeByConfiguration[request.BuildRequest.ConfigurationId] = accumulatedTime + request.GetTimeSpentInState(SchedulableRequestState.Executing).TotalMilliseconds;
foreach (SchedulableRequest childRequest in _schedulingData.GetRequestsByHierarchy(request))
{
RecursiveAccumulateConfigurationTimes(childRequest, accumulatedTimeByConfiguration);
}
}
/// <summary>
/// Detects a circular dependency where the blocking request is in our direct ancestor chain.
/// </summary>
private void DetectDirectCircularDependency(SchedulableRequest blockingRequest)
{
// A circular dependency occurs when this project (or any of its ancestors) has the same global request id as the
// blocking request.
List<SchedulableRequest> ancestors = new List<SchedulableRequest>(16);
SchedulableRequest currentRequest = this;
do
{
ancestors.Add(currentRequest);
if (currentRequest.BuildRequest.GlobalRequestId == blockingRequest.BuildRequest.GlobalRequestId)
{
// We are directly conflicting with an instance of ourselves.
CleanupForCircularDependencyAndThrow(blockingRequest, ancestors);
}
currentRequest = currentRequest.Parent;
}
while (currentRequest != null);
}
/// <summary>
/// Writes out all of the dependencies for a specified request, recursively.
/// </summary>
private void RecursiveWriteDependencies(StreamWriter file, SchedulableRequest request)
{
file.Write(request.BuildRequest.ConfigurationId);
foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request))
{
file.Write(" {0}", child.BuildRequest.ConfigurationId);
}
file.WriteLine();
foreach (SchedulableRequest child in _schedulingData.GetRequestsByHierarchy(request))
{
RecursiveWriteDependencies(file, child);
}
}
/// <summary>
/// Retrieves a set of build requests which have the specified parent. If root is null, this will retrieve all of the
/// top-level requests.
/// </summary>
public IEnumerable<SchedulableRequest> GetRequestsByHierarchy(SchedulableRequest root)
{
if (root == null)
{
// Retrieve all requests which are roots of the tree.
List<SchedulableRequest> roots = new List<SchedulableRequest>();
foreach (SchedulableRequest key in _buildHierarchy.Keys)
{
if (key.Parent == null)
{
roots.Add(key);
}
}
return roots;
}
return _buildHierarchy[root];
}
/// <summary>
/// Constructor.
/// </summary>
public SchedulableRequest(SchedulingData collection, BuildRequest request, SchedulableRequest parent)
{
ErrorUtilities.VerifyThrowArgumentNull(collection, "collection");
ErrorUtilities.VerifyThrowArgumentNull(request, "request");
ErrorUtilities.VerifyThrow((parent == null) || (parent._schedulingData == collection), "Parent request does not belong to the same collection.");
_schedulingData = collection;
_request = request;
_parent = parent;
_assignedNodeId = -1;
_requestsWeAreBlockedBy = new Dictionary<BlockingRequestKey, SchedulableRequest>();
_requestsWeAreBlocking = new HashSet<SchedulableRequest>();
_timeRecords = new Dictionary<SchedulableRequestState, ScheduleTimeRecord>(5);
_timeRecords[SchedulableRequestState.Unscheduled] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Blocked] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Yielding] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Executing] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Ready] = new ScheduleTimeRecord();
_timeRecords[SchedulableRequestState.Completed] = new ScheduleTimeRecord();
ChangeToState(SchedulableRequestState.Unscheduled);
}
/// <summary>
/// Determines if the specified request is currently scheduled.
/// </summary>
public bool IsRequestScheduled(SchedulableRequest request)
{
return(InternalGetScheduledRequestByGlobalRequestId(request.BuildRequest.GlobalRequestId) != null);
}
/// <summary>
/// Detects a circular dependency. Throws a CircularDependencyException if one exists. Circular dependencies can occur
/// under the following conditions:
/// 1. If the blocking request's global request ID appears in the ancestor chain (Direct).
/// 2. If a request appears in the ancestor chain and has a different global request ID but has an active target that
/// matches one of the targets specified in the blocking request (Direct).
/// 3. If the blocking request exists elsewhere as a blocked request with the same global request ID, and one of its children
/// (recursively) matches this request's global request ID (Indirect).
/// 4. If the blocking request's configuration is part of another request elsewhere which is also blocked, and that request
/// is building targets this blocking request is building, and one of that blocked request's children (recursively)
/// matches this request's global request ID (Indirect).
/// </summary>
private void DetectCircularDependency(SchedulableRequest blockingRequest)
{
DetectDirectCircularDependency(blockingRequest);
DetectIndirectCircularDependency(blockingRequest);
}
/// <summary>
/// Constructor.
/// </summary>
public SchedulingEvent(DateTime eventTime, SchedulableRequest request, SchedulableRequestState oldState, SchedulableRequestState newState)
{
_eventTime = eventTime;
_request = request;
_oldState = oldState;
_newState = newState;
}
/// <summary>
/// Build our ancestor list then throw the circular dependency error.
/// </summary>
private void ThrowIndirectCircularDependency(SchedulableRequest blockingRequest, SchedulableRequest requestToEvaluate)
{
// We found a request which has the same global request ID as us in a chain which leads from the (already blocked) request
// which is trying to block us. Calculate its list of ancestors by walking up the parent list.
List<SchedulableRequest> ancestors = new List<SchedulableRequest>(16);
while (requestToEvaluate.Parent != null)
{
ancestors.Add(requestToEvaluate.Parent);
requestToEvaluate = requestToEvaluate.Parent;
}
ancestors.Reverse(); // Because the list should be in the order from root to child.
CleanupForCircularDependencyAndThrow(blockingRequest, ancestors);
}
/// <summary>
/// Updates the state of the specified request.
/// </summary>
public void UpdateFromState(SchedulableRequest request, SchedulableRequestState previousState)
{
// Remove from its old collection
switch (previousState)
{
case SchedulableRequestState.Blocked:
_blockedRequests.Remove(request.BuildRequest.GlobalRequestId);
break;
case SchedulableRequestState.Yielding:
_yieldingRequests.Remove(request.BuildRequest.GlobalRequestId);
break;
case SchedulableRequestState.Completed:
ErrorUtilities.ThrowInternalError("Should not be updating a request after it has reached the Completed state.");
break;
case SchedulableRequestState.Executing:
_executingRequests.Remove(request.BuildRequest.GlobalRequestId);
_executingRequestByNode[request.AssignedNode] = null;
break;
case SchedulableRequestState.Ready:
_readyRequests.Remove(request.BuildRequest.GlobalRequestId);
_readyRequestsByNode[request.AssignedNode].Remove(request);
break;
case SchedulableRequestState.Unscheduled:
LinkedListNode <SchedulableRequest> requestNode = _unscheduledRequestNodesByRequest[request];
_unscheduledRequestNodesByRequest.Remove(request);
_unscheduledRequests.Remove(requestNode);
if (request.State != SchedulableRequestState.Completed)
{
// Map the request to the node.
HashSet <SchedulableRequest> requestsAssignedToNode;
if (!_scheduledRequestsByNode.TryGetValue(request.AssignedNode, out requestsAssignedToNode))
{
requestsAssignedToNode = new HashSet <SchedulableRequest>();
_scheduledRequestsByNode[request.AssignedNode] = requestsAssignedToNode;
}
ErrorUtilities.VerifyThrow(!requestsAssignedToNode.Contains(request), "Request {0} is already scheduled to node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode);
requestsAssignedToNode.Add(request);
// Map the configuration to the node.
HashSet <int> configurationsAssignedToNode;
if (!_configurationsByNode.TryGetValue(request.AssignedNode, out configurationsAssignedToNode))
{
configurationsAssignedToNode = new HashSet <int>();
_configurationsByNode[request.AssignedNode] = configurationsAssignedToNode;
}
if (!configurationsAssignedToNode.Contains(request.BuildRequest.ConfigurationId))
{
configurationsAssignedToNode.Add(request.BuildRequest.ConfigurationId);
}
}
break;
}
// Add it to its new location
switch (request.State)
{
case SchedulableRequestState.Blocked:
ErrorUtilities.VerifyThrow(!_blockedRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already blocked!");
_blockedRequests[request.BuildRequest.GlobalRequestId] = request;
break;
case SchedulableRequestState.Yielding:
ErrorUtilities.VerifyThrow(!_yieldingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already yielded!");
_yieldingRequests[request.BuildRequest.GlobalRequestId] = request;
break;
case SchedulableRequestState.Completed:
ErrorUtilities.VerifyThrow(_configurationToRequests.ContainsKey(request.BuildRequest.ConfigurationId), "Configuration {0} never had requests assigned to it.", request.BuildRequest.ConfigurationId);
ErrorUtilities.VerifyThrow(_configurationToRequests[request.BuildRequest.ConfigurationId].Count > 0, "Configuration {0} has no requests assigned to it.", request.BuildRequest.ConfigurationId);
_configurationToRequests[request.BuildRequest.ConfigurationId].Remove(request);
if (_scheduledRequestsByNode.ContainsKey(request.AssignedNode))
{
_scheduledRequestsByNode[request.AssignedNode].Remove(request);
}
request.EndTime = EventTime;
break;
case SchedulableRequestState.Executing:
ErrorUtilities.VerifyThrow(!_executingRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already executing!");
ErrorUtilities.VerifyThrow(!_executingRequestByNode.ContainsKey(request.AssignedNode) || _executingRequestByNode[request.AssignedNode] == null, "Node {0} is currently executing a request.", request.AssignedNode);
_executingRequests[request.BuildRequest.GlobalRequestId] = request;
_executingRequestByNode[request.AssignedNode] = request;
_configurationToNode[request.BuildRequest.ConfigurationId] = request.AssignedNode;
if (previousState == SchedulableRequestState.Unscheduled)
{
request.StartTime = EventTime;
}
break;
case SchedulableRequestState.Ready:
ErrorUtilities.VerifyThrow(!_readyRequests.ContainsKey(request.BuildRequest.GlobalRequestId), "Request with global id {0} is already ready!");
_readyRequests[request.BuildRequest.GlobalRequestId] = request;
HashSet <SchedulableRequest> readyRequestsOnNode;
if (!_readyRequestsByNode.TryGetValue(request.AssignedNode, out readyRequestsOnNode))
{
readyRequestsOnNode = new HashSet <SchedulableRequest>();
_readyRequestsByNode[request.AssignedNode] = readyRequestsOnNode;
}
ErrorUtilities.VerifyThrow(!readyRequestsOnNode.Contains(request), "Request with global id {0} is already marked as ready on node {1}", request.BuildRequest.GlobalRequestId, request.AssignedNode);
readyRequestsOnNode.Add(request);
break;
case SchedulableRequestState.Unscheduled:
ErrorUtilities.ThrowInternalError("Request with global id {0} cannot transition to the Unscheduled state", request.BuildRequest.GlobalRequestId);
break;
}
_buildEvents.Add(new SchedulingEvent(EventTime, request, previousState, request.State));
}
/// <summary>
/// Removes associations with all blocking requests and throws an exception.
/// </summary>
private void CleanupForCircularDependencyAndThrow(SchedulableRequest requestCausingFailure, List<SchedulableRequest> ancestors)
{
if (_requestsWeAreBlockedBy.Count != 0)
{
List<SchedulableRequest> tempRequests = new List<SchedulableRequest>(_requestsWeAreBlockedBy.Values);
foreach (SchedulableRequest requestWeAreBlockedBy in tempRequests)
{
BlockingRequestKey key = new BlockingRequestKey(requestWeAreBlockedBy.BuildRequest);
DisconnectRequestWeAreBlockedBy(key);
}
}
else
{
ChangeToState(SchedulableRequestState.Ready);
}
_activeTargetsWhenBlocked = null;
// The blocking request itself is no longer valid if it was unscheduled.
if (requestCausingFailure.State == SchedulableRequestState.Unscheduled)
{
requestCausingFailure.Delete();
}
throw new SchedulerCircularDependencyException(requestCausingFailure.BuildRequest, ancestors);
}
/// <summary>
/// Gets the name of the plan file for a specified submission.
/// </summary>
private string GetPlanName(SchedulableRequest rootRequest)
{
if (rootRequest == null)
{
return null;
}
return _configCache[rootRequest.BuildRequest.ConfigurationId].ProjectFullPath + ".buildplan";
}