public void ScheduleProcess(ClusterInterface.ISchedulerProcess ip,
List <ClusterInterface.Affinity> affinities,
ClusterInterface.RunProcess onScheduled)
{
Process process = ip as Process;
Task.Run(() => ScheduleProcessInternal(process, affinities, onScheduled));
}
public void SetCallback(ClusterInterface.RunProcess callback)
{
OnScheduledCallback = callback;
}
private async void ScheduleProcessInternal(Process process, List <ClusterInterface.Affinity> affinities,
ClusterInterface.RunProcess callback)
{
logger.Log("Scheduling process " + process.Id);
process.SetCallback(callback);
Task rackBlocker;
Task clusterBlocker;
lock (this)
{
rackBlocker = Task.WhenAny(flusher, Task.Delay(rackDelay));
clusterBlocker = Task.WhenAny(flusher, Task.Delay(clusterDelay));
}
bool isHardConstraint = affinities.Aggregate(false, (a, b) => a || b.isHardContraint);
if (isHardConstraint)
{
// the constraint generator should have intersected the hard constraint into a single one
Debug.Assert(affinities.Count() == 1);
logger.Log("Process " + process.Id + " has a hard constraint");
}
var allAffinities = affinities.SelectMany(a => a.affinities).Distinct();
var computerAffinities = allAffinities.Where(a => a.level == ClusterInterface.AffinityResourceLevel.Host);
bool addedAny = false;
// get a snapshot of available computers
Dictionary <string, List <Computer> > localitySnapshot = new Dictionary <string, List <Computer> >();
lock (localities)
{
foreach (var c in localities)
{
localitySnapshot.Add(c.Key, c.Value);
}
}
if (localitySnapshot.Count == 0)
{
await process.OnScheduled(null, -1, null, "No cluster computers available");
return;
}
var racksUsed = new List <string>();
foreach (var a in computerAffinities)
{
List <Computer> cl;
if (localitySnapshot.TryGetValue(a.locality, out cl))
{
addedAny = true;
logger.Log("Adding Process " + process.Id + " to queues for computers with locality " + a.locality);
foreach (var c in cl)
{
logger.Log("Adding Process " + process.Id + " to queue for computer " + c.Name);
if (c.LocalQueue.AddProcess(process))
{
// this returns true if p has been matched to a computer, in which case we
// can stop adding it to queues
logger.Log("Process " + process.Id + " claimed by computer " + c.Name);
return;
}
}
// remember the rack this computer was in, to include it for soft affinities below
racksUsed.Add(cl.First().RackName);
}
}
if (addedAny)
{
// hacky delay scheduling; wait until the upper level has finished adding processes in
// the current stage, or some time has passed, before relaxing affinities if the process
// had affinities for particular computers
logger.Log("Process " + process.Id + " delay scheduling for rack");
await rackBlocker;
}
// reset flag before adding to racks
addedAny = false;
// get a snapshot of available racks
Dictionary <string, Rack> rackSnapshot = new Dictionary <string, Rack>();
lock (racks)
{
foreach (var r in racks)
{
rackSnapshot.Add(r.Key, r.Value);
}
}
var rackAffinities = allAffinities.Where(a => a.level == ClusterInterface.AffinityResourceLevel.Rack).Select(a => a.locality).Distinct();
if (!isHardConstraint)
{
rackAffinities = rackAffinities.Concat(racksUsed).Distinct();
}
foreach (var a in rackAffinities)
{
Rack r;
if (rackSnapshot.TryGetValue(a, out r))
{
addedAny = true;
logger.Log("Adding Process " + process.Id + " to queue for rack " + a);
if (r.queue.AddProcess(process))
{
// this returns true if p has been matched to a computer, in which case we
// can stop adding it to queues
logger.Log("Process " + process.Id + " claimed by rack " + a);
return;
}
}
}
if (isHardConstraint)
{
// let the process know it won't get added to any more queues. This will signal the
// upper layer if it didn't get added to any queues
process.FinishedScheduling();
return;
}
if (addedAny)
{
// hacky delay scheduling; wait until the upper level has finished adding processes in
// the current stage, or some time has passed, before relaxing affinities if the process
// had affinities for particular racks
logger.Log("Process " + process.Id + " delay scheduling for cluster");
await clusterBlocker;
}
logger.Log("Adding Process " + process.Id + " to queue for cluster");
clusterQueue.AddProcess(process);
// let the process know it won't get added to any more queues
process.FinishedScheduling();
}
public void SetCallback(ClusterInterface.RunProcess callback)
{
OnScheduledCallback = callback;
}