public bool Add(Dispatcher d)
{
lock (SyncRoot)
{
// We never want to have two dispatchers for the same node
// since we don't support oversubscription for vertex nodes
Dispatcher dummy = null;
if (!GetByNodeName(d.NodeName, out dummy))
{
m_dispatcherTable.Add(d);
return true;
}
else
{
return false;
}
}
}
public bool Remove(Dispatcher d)
{
lock (SyncRoot)
{
return m_dispatcherTable.Remove(d);
}
}
public bool TryReserveDispatcher(string node, out Dispatcher dispatcher)
{
lock (SyncRoot)
{
Dispatcher d = null;
if (GetByNodeName(node, out d))
{
if (d.Reserve())
{
dispatcher = d;
return true;
}
}
}
dispatcher = null;
return false;
}
public bool GetByTaskId(int taskId, out Dispatcher d)
{
lock (SyncRoot)
{
d = m_dispatcherTable.Find(x => x.TaskId == taskId);
}
return (d != null);
}
public bool GetByNodeName(string node, out Dispatcher d)
{
lock (SyncRoot)
{
d = m_dispatcherTable.Find(x => String.Compare(x.NodeName, node, StringComparison.OrdinalIgnoreCase) == 0);
}
return (d != null);
}
private bool FindNodeForRequest(ScheduleProcessRequest req, out Dispatcher dispatcher)
{
dispatcher = null;
if (req.HardAffinity != null)
{
if (dispatcherPool.TryReserveDispatcher(req.HardAffinity, out dispatcher))
{
return true;
}
else
{
return false;
}
}
else
{
// First try soft affinity in decreasing order (assumes Soft Affinity list in req is sorted descending by weight)
// Keep a map of the nodes we've already tried, because Dryad adds each affinity twice
// once for the node and once for the "pod"
Dictionary<string, bool> attemptedNodes = new Dictionary<string, bool>();
int count = 0;
for (int i = 0; i < req.AffinityCount; i++)
{
if (attemptedNodes.ContainsKey(req.AffinityAt(i).Node.ToUpper()))
{
continue;
}
attemptedNodes.Add(req.AffinityAt(i).Node.ToUpper(), true);
count++;
if (dispatcherPool.TryReserveDispatcher(req.AffinityAt(i).Node, out dispatcher))
{
DryadLogger.LogDebug("Find Node For Request", "process {0} satisfied affinity constraint: node {1}, weight {2}", req.Id, req.AffinityAt(i).Node, req.AffinityAt(i).Weight);
return true;
}
DryadLogger.LogDebug("Find Node For Request", "process {0} did not satisfy affinity constraint: node {1}, weight {2}", req.Id, req.AffinityAt(i).Node, req.AffinityAt(i).Weight);
}
// If we get this far and AffinityCount > 0, then we failed to satisfy the affinity constraints
// log a message so we can more easily detect this situation
if (count > 0)
{
DryadLogger.LogInformation("Find Node For Request", "process {0} failed to satisfy any of {1} affinity constraints", req.Id, count);
}
// Finally try any available node
lock (dispatcherPool.SyncRoot)
{
foreach (Dispatcher d in dispatcherPool)
{
if (req.CanRunOnNode(d.NodeName))
{
if (d.Reserve())
{
dispatcher = d;
return true;
}
}
}
}
}
return false;
}
/// <summary>
/// Create a new dispatcher and add to the good dispatcher pool.
/// </summary>
/// <param name="taskid">HPC Task Id</param>
/// <param name="node">Name of node this dispatcher is for</param>
/// <param name="state">State of task when dispatcher is created (always Running now)</param>
/// <returns>Dispatcher that was added, or null if a dispatcher already exists in the good pool for specified node</returns>
private Dispatcher AddDispatcher(int taskid, string node, VertexTaskState state)
{
VertexComputeNode cn = new VertexComputeNode();
cn.instanceId = taskid;
cn.ComputeNode = node;
cn.State = state;
Dispatcher d = new Dispatcher(schedulerHelper, cn);
d.FaultedEvent += new DispatcherFaultedEventHandler(OnDispatcherFaulted);
if (!dispatcherPool.Add(d))
{
// There's already a dispatcher for this node
d.Dispose();
d = null;
}
return d;
}
private bool ScheduleProcess(ScheduleProcessRequest request, Dispatcher dispatcher)
{
lock (processTable.SyncRoot)
{
lock (this.processTable[request.Id].SyncRoot)
{
processTable[request.Id].Dispatcher = dispatcher;
}
}
if (dispatcher.ScheduleProcess(replyUri, request, new AsyncCallback(this.ScheduleProcessCallback)))
{
DryadLogger.LogInformation("Schedule Process", "Began asynchronous scheduling of process {0} on node '{1}': '{2}'", request.Id, dispatcher.NodeName, request.CommandLine);
return true;
}
else
{
DryadLogger.LogWarning("Schedule Process", "Failed to begin asynchronous scheduling of process {0} on node '{1}'", request.Id, dispatcher.NodeName);
return false;
}
}
