public void InitClusters()
{
this.clusterFacade.ClusterSetting = this.clusterSetting;
this.clusterFacade.DataSet = this.dataset;
this.inputCluster = this.clusterFacade.getInputCluster(this.appConfig);
this.outputCluster = this.clusterFacade.getOutputCluster(this.appConfig);
}
public LocalScheduler(ClusterInterface.ILogger l)
{
logger = l;
computers = new Dictionary<string, Computer>();
localities = new Dictionary<string, List<Computer>>();
racks = new Dictionary<string, Rack>();
clusterQueue = new ProcessQueue();
flusher = new Task(() => { });
clusterInterface = new PeloponneseInterface();
dummyCancelComputer = new Computer("dummy for canceling", "nowhere", "no rack", null, null,
"no server", "no server", "no directory", logger);
l.Log("LocalScheduler created");
}
private static LocalScheduler Create(ClusterInterface.ILogger logger)
{
return new LocalScheduler(logger);
}
public void CancelProcess(ClusterInterface.ISchedulerProcess ip)
{
Process process = ip as Process;
lock (process)
{
if (process.Unclaimed)
{
// it is still sitting in scheduling queues; unblock it with a dummy process
process.Claim(dummyCancelComputer);
}
}
}
public void ScheduleProcess(ClusterInterface.ISchedulerProcess ip,
List<ClusterInterface.Affinity> affinities,
ClusterInterface.RunProcess onScheduled)
{
Process process = ip as Process;
Task.Run(() => ScheduleProcessInternal(process, affinities, onScheduled));
}
private async void ScheduleProcessInternal(Process process, List<ClusterInterface.Affinity> affinities,
ClusterInterface.RunProcess callback)
{
logger.Log("Scheduling process " + process.Id);
process.SetCallback(callback);
Task rackBlocker;
Task clusterBlocker;
lock (this)
{
rackBlocker = Task.WhenAny(flusher, Task.Delay(rackDelay));
clusterBlocker = Task.WhenAny(flusher, Task.Delay(clusterDelay));
}
bool isHardConstraint = affinities.Aggregate(false, (a, b) => a || b.isHardContraint);
if (isHardConstraint)
{
// the constraint generator should have intersected the hard constraint into a single one
Debug.Assert(affinities.Count() == 1);
logger.Log("Process " + process.Id + " has a hard constraint");
}
var allAffinities = affinities.SelectMany(a => a.affinities).Distinct();
var computerAffinities = allAffinities.Where(a => a.level == ClusterInterface.AffinityResourceLevel.Host);
bool addedAny = false;
// get a snapshot of available computers
Dictionary<string, List<Computer>> localitySnapshot = new Dictionary<string,List<Computer>>();
lock (localities)
{
foreach (var c in localities)
{
localitySnapshot.Add(c.Key, c.Value);
}
}
if (localitySnapshot.Count == 0)
{
await process.OnScheduled(null, -1, null, "No cluster computers available");
return;
}
var racksUsed = new List<string>();
foreach (var a in computerAffinities)
{
List<Computer> cl;
if (localitySnapshot.TryGetValue(a.locality, out cl))
{
addedAny = true;
logger.Log("Adding Process " + process.Id + " to queues for computers with locality " + a.locality);
foreach (var c in cl)
{
logger.Log("Adding Process " + process.Id + " to queue for computer " + c.Name);
if (c.LocalQueue.AddProcess(process))
{
// this returns true if p has been matched to a computer, in which case we
// can stop adding it to queues
logger.Log("Process " + process.Id + " claimed by computer " + c.Name);
return;
}
}
// remember the rack this computer was in, to include it for soft affinities below
racksUsed.Add(cl.First().RackName);
}
}
if (addedAny)
{
// hacky delay scheduling; wait until the upper level has finished adding processes in
// the current stage, or some time has passed, before relaxing affinities if the process
// had affinities for particular computers
logger.Log("Process " + process.Id + " delay scheduling for rack");
await rackBlocker;
}
// reset flag before adding to racks
addedAny = false;
// get a snapshot of available racks
Dictionary<string, Rack> rackSnapshot = new Dictionary<string, Rack>();
lock (racks)
{
foreach (var r in racks)
{
rackSnapshot.Add(r.Key, r.Value);
}
}
var rackAffinities = allAffinities.Where(a => a.level == ClusterInterface.AffinityResourceLevel.Rack).Select(a => a.locality).Distinct();
if (!isHardConstraint)
{
rackAffinities = rackAffinities.Concat(racksUsed).Distinct();
}
foreach (var a in rackAffinities)
{
Rack r;
if (rackSnapshot.TryGetValue(a, out r))
{
addedAny = true;
logger.Log("Adding Process " + process.Id + " to queue for rack " + a);
if (r.queue.AddProcess(process))
{
// this returns true if p has been matched to a computer, in which case we
// can stop adding it to queues
logger.Log("Process " + process.Id + " claimed by rack " + a);
return;
}
}
}
if (isHardConstraint)
{
// let the process know it won't get added to any more queues. This will signal the
// upper layer if it didn't get added to any queues
process.FinishedScheduling();
return;
}
if (addedAny)
{
// hacky delay scheduling; wait until the upper level has finished adding processes in
// the current stage, or some time has passed, before relaxing affinities if the process
// had affinities for particular racks
logger.Log("Process " + process.Id + " delay scheduling for cluster");
await clusterBlocker;
}
logger.Log("Adding Process " + process.Id + " to queue for cluster");
clusterQueue.AddProcess(process);
// let the process know it won't get added to any more queues
process.FinishedScheduling();
}
public string initMembership(ClusterInterface inputCluster, ClusterInterface outputCluster)
{
double[,] inputCentroids = inputCluster.Centroids;
double[,] inputWidths = inputCluster.Widths;
double[,] outputCentroids = outputCluster.Centroids;
double[,] outputWidths = outputCluster.Widths;
string result = "Condition Layer:\n";
int k = 0;
for (int i=0; i<this.numberOfInputs; i++)
{
for (int j=0; j<this.inputClusterSize; j++)
{
conditionLayer[k].setCentroidAndWidth(inputCentroids[j, i], inputWidths[j, i]);
result += String.Format("centroid:{0:F3}\twidth:{1:F3}\n", conditionLayer[k].Centroid, conditionLayer[k].Width);
k++;
}
}
result += "\nConsequence Layer:\n";
k = 0;
for (int i=0; i<this.numberOfOutputs; i++)
{
for (int j=0; j<this.outputClusterSize; j++)
{
consequenceLayer[k].setCentroidAndWidth(outputCentroids[j, i], outputWidths[j, i]);
result += String.Format("centroid:{0:F3}\twidth:{1:F3}\n", consequenceLayer[k].Centroid, consequenceLayer[k].Width);
k++;
}
}
return result;
}
public bool Initialize(LocalScheduler p, ClusterInterface.ILogger l)
{
parent = p;
logger = l;
epoch = 0;
version = 0;
targetNumberOfWorkers = -1;
knownWorkers = new Dictionary<string, string>();
reasonableReached = new TaskCompletionSource<bool>();
shutdownTask = new TaskCompletionSource<XContainer>();
waitingForComputer = new List<Task>();
exited = new TaskCompletionSource<bool>();
jobGuid = Environment.GetEnvironmentVariable(Constants.EnvJobGuid);
if (jobGuid == null)
{
logger.Log("Can't find environment variable " + Constants.EnvJobGuid + ": exiting");
return false;
}
serverAddress = Environment.GetEnvironmentVariable(Constants.EnvManagerServerUri);
if (serverAddress == null)
{
logger.Log("Can't find environment variable " + Constants.EnvManagerServerUri + ": exiting");
return false;
}
var groupName = Environment.GetEnvironmentVariable(Constants.EnvProcessGroup);
if (groupName == null)
{
logger.Log("Can't find environment variable " + Constants.EnvProcessGroup + ": exiting");
return false;
}
var procIdentifier = Environment.GetEnvironmentVariable(Constants.EnvProcessIdentifier);
if (procIdentifier == null)
{
logger.Log("Can't find environment variable " + Constants.EnvProcessIdentifier + ": exiting");
return false;
}
var element = new XElement("ProcessDetails");
var status = element.ToString();
string registration = String.Format("{0}register?guid={1}&group={2}&identifier={3}", serverAddress, jobGuid, groupName, procIdentifier);
IHttpRequest request = ClusterInterface.HttpClient.Create(registration);
request.Timeout = 30 * 1000; // if this doesn't come back quickly, we'll get an exception and quit
request.Method = "POST";
try
{
using (Stream upload = request.GetRequestStream())
{
using (StreamWriter sw = new StreamWriter(upload))
{
sw.Write(status);
}
}
using (IHttpResponse response = request.GetResponse())
{
logger.Log("Server registration succeeded");
return true;
}
}
catch (NotHttpException e)
{
// if this failed, there's nothing much more we can do
logger.Log("Server registration failed message " + e.Message + " status " + e.Response.StatusCode + ": " + e.Response.StatusDescription);
return false;
}
catch (Exception e)
{
// if this failed, there's nothing much more we can do
logger.Log("Server registration failed message " + e.Message);
return false;
}
}
public Task OnScheduled(ClusterInterface.IComputer computer, int processId, Task blocker, string errorReason)
{
return OnScheduledCallback(computer, processId, blocker, errorReason);
}
public void SetCallback(ClusterInterface.RunProcess callback)
{
OnScheduledCallback = callback;
}
