private void notifyWorkersAboutFailedNode(int failedNodeId)
{
IWorkerTracker worker = null;
for (int i = 0; i < existingWorkerMap.Count; i++)
{
KeyValuePair <Int32, WorkerDetails> entry = existingWorkerMap.ElementAt(i);
try
{
lock (workerProxyMap)
{
if (!workerProxyMap.ContainsKey(entry.Key))
{
string url = entry.Value.Nodeurl;
worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url);
if (worker != null)
{
workerProxyMap.Add(entry.Key, worker);
}
}
else
{
worker = workerProxyMap[entry.Key];
}
}
worker.removeFailedNode(failedNodeId);
}
catch (Exception ex)
{
Common.Logger().LogError("job failure notificatin of node ", string.Empty, string.Empty);
}
}
}
private void notifyWorkersAboutUnfreezed(int nodeId, String nodeURL)
{
IWorkerTracker worker = null;
for (int i = 0; i < existingWorkerMap.Count; i++)
{
KeyValuePair <Int32, WorkerDetails> entry = existingWorkerMap.ElementAt(i);
try
{
lock (workerProxyMap)
{
if (!workerProxyMap.ContainsKey(entry.Key))
{
string url = entry.Value.Nodeurl;
worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url);
if (worker != null)
{
workerProxyMap.Add(entry.Key, worker);
}
}
else
{
worker = workerProxyMap[entry.Key];
}
}
worker.addUnfreezedNode(nodeId, nodeURL);
}
catch (Exception ex)
{
Common.Logger().LogError("Unable to notify others about unfreeze by Job Tracker", string.Empty, string.Empty);
}
}
}
internal void notifyWorkersForJobStart()
{
IWorkerTracker worker = null;
for (int i = 0; i < existingWorkerMap.Count; i++)
{
KeyValuePair <Int32, WorkerDetails> entry = existingWorkerMap.ElementAt(i);
try
{
lock (workerProxyMap)
{
if (!workerProxyMap.ContainsKey(entry.Key))
{
string url = entry.Value.Nodeurl;
worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url);
}
else
{
worker = workerProxyMap[entry.Key];
}
}
worker.receiveNewJob();
}
catch (Exception ex)
{
Common.Logger().LogError("Unable to notify others about unfreeze by Job Tracker", string.Empty, string.Empty);
}
}
}
internal void notifyExistingWorkers(int workerId, String newWorkerURL, Dictionary <Int32, WorkerDetails> existingWorkerList)
{
foreach (KeyValuePair <Int32, WorkerDetails> entry in existingWorkerList)
{
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entry.Value.Nodeurl);
worker.addNewWorker(workerId, newWorkerURL);
}
}
public void notifyBackupJobtrackerUrl(string url, Dictionary <Int32, WorkerDetails> existingWorkerList)
{
foreach (KeyValuePair <Int32, WorkerDetails> entry in existingWorkerList)
{
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entry.Value.Nodeurl);
worker.SetBackupJobTrackerUrl(url);
}
}
public void SendTaskCopyToBackupTracker(string url, Dictionary <int, Task> taskList, string clientUrl)
{
IWorkerTracker bkpTracker = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
url);
bkpTracker.SetCopyOfTasks(taskList, clientUrl);
}
public void TrackerStabilized(Dictionary <Int32, WorkerDetails> existingWorkerList)
{
foreach (KeyValuePair <Int32, WorkerDetails> entry in existingWorkerList)
{
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entry.Value.Nodeurl);
worker.TrackerStabilized();
}
}
internal void notifyRecoveryNodeToTracker(int workerID, string nodeURL, List <int> processingSplits, List <int> alreadySentSplits)
{
if (trackerProxy == null)
{
trackerProxy = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
Worker.JOBTRACKER_URL);
}
trackerProxy.notifyWorkerRecovery(workerID, nodeURL, processingSplits, alreadySentSplits);
}
public void FeedNewTracker(int workerID, String workerURL, List <int> processingSplits, List <int> alreadySentSplits)
{
if (!WorkerTask.IS_WORKER_FREEZED)
{
trackerProxy = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
Worker.BKP_JOBTRACKER_URL);
trackerProxy.ChangeTracker(workerID, workerURL, processingSplits, alreadySentSplits);
}
}
internal Dictionary <StatusType, List <int> > notifyTrackerOnUnfreeze(Dictionary <StatusType, List <int> > freezedWorkerStatus, int nodeId, string nodeURL)
{
if (trackerProxy == null)
{
trackerProxy = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
Worker.JOBTRACKER_URL);
}
Dictionary <StatusType, List <int> > updatedStatus = trackerProxy.receiveFreezedWorkerStatus(freezedWorkerStatus, nodeId, nodeURL);
return(updatedStatus);
}
internal void hasThresholdReached(int nodeId)
{
if (!WorkerTask.IS_WORKER_FREEZED)
{
if (trackerProxy == null)
{
trackerProxy = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
Worker.JOBTRACKER_URL);
}
Common.Logger().LogInfo("requesting task in threashold reached", string.Empty, string.Empty);
trackerProxy.readyForNewTask(nodeId);
}
}
public void sendStatusUpdatesToTracker(Status status, WorkerTask workerTask)
{
try
{
workerTask.checkWorkerFreezed();
CheckSystemStability();
if (trackerProxy == null)
{
trackerProxy = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
Worker.JOBTRACKER_URL);
}
trackerProxy.receiveStatus(status);
}
catch (Exception ex)
{
Common.Logger().LogError("unable to send status updates by " + status.NodeId + " for percentage " + status.PercentageCompleted, string.Empty, string.Empty);
}
}
internal void notifyTaskCompletedEvent(int workerId, int splitId)
{
try
{
CheckSystemStability();
Common.Logger().LogInfo("Notifying task completed event for split " + splitId, string.Empty, string.Empty);
if (trackerProxy == null)
{
trackerProxy = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
Worker.JOBTRACKER_URL);
}
trackerProxy.taskCompleted(workerId, splitId, Worker.serviceUrl);
}
catch (Exception ex)
{
Common.Logger().LogInfo("exception thrown while sending copleted event ", string.Empty, string.Empty);
}
}
public void submitTask(String entryUrl, string inputFile, string outputDir, int splits, string mapperFunctionName, string dllPath)
{
hasJobCompleted = false;
startTime = DateTime.Now;
this.inputFilePath = inputFile;
this.outputDir = outputDir;
this.mapperName = mapperFunctionName;
this.dllPath = dllPath;
// FileInfo f = new FileInfo(inputFilePath);
// long s1 = f.Length;
//JobMetadata jobDetails = new JobMetadata(s1, splits, url);
byte[] input = File.ReadAllBytes(inputFilePath);
JobMetadata jobDetails = new JobMetadata(input.Length, splits, url);
contactingWorker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entryUrl);
contactingWorker.receiveJobRequest(jobDetails);//calling the job tracker
}
/*important event so if not send add to list back*/
internal void notifyResultsSentToClientEvent(int workerId, TaskResult taskResult, WorkerTask workerTask)
{
try
{
workerTask.checkWorkerFreezed();
CheckSystemStability();
if (trackerProxy == null)
{
trackerProxy = (IWorkerTracker)Activator.GetObject(
typeof(IWorkerTracker),
Worker.JOBTRACKER_URL);
}
Common.Logger().LogInfo("Sent to taskCompleted by Worker ID = " + workerId + " Split Id= " + taskResult.SplitId, string.Empty, string.Empty);
trackerProxy.taskCompleted(workerId, taskResult.SplitId, Worker.serviceUrl);
}
catch (Exception ex)
{
workerTask.addTaskToTaskResults(taskResult);
Common.Logger().LogInfo("exception thrown while sending copleted event ", string.Empty, string.Empty);
Common.Logger().LogInfo(ex.Message, ex.StackTrace, string.Empty);
}
}
/*going to replace one slow task*/
private void ReplaceSlowTasks(int nodeId)
{
if (!HasPendingTasks())
{
WorkerDetails bestWorker = null;
FileSplitMetadata splitMetadata = null;
int splitId = 0;
lock (taskList)
{
foreach (var pair in taskList)
{
/* later change this to have from suspended as well */
if (pair.Value.StatusType == StatusType.INPROGRESS)
{
splitId = pair.Key;
bestWorker = GetBestWorker(pair.Value, nodeId);
if (bestWorker != null)
{
bestWorker.State = WorkerState.ABOUT_TO_BUSY;
string url = existingWorkerMap[taskList[splitId].WorkerId].Nodeurl;
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url);
worker.suspendTask(splitId);
//reschedule
splitMetadata = pair.Value.SplitMetadata;
taskList[splitId].StatusType = StatusType.INPROGRESS;
taskList[splitId].WorkerId = bestWorker.Nodeid;
sendTaskToWorker(bestWorker.Nodeid, splitMetadata);
}
}
}
}
}
else
{
readyForNewTask(nodeId);
}
}
private void sendTaskToWorker(int nodeId, FileSplitMetadata splitMetadata)
{
if (!isTrackerFreezed)
{
IWorkerTracker worker = null;
try
{
lock (workerProxyMap)
{
if (!workerProxyMap.ContainsKey(nodeId))
{
string url = existingWorkerMap[nodeId].Nodeurl;
worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url);
if (worker != null)
{
workerProxyMap.Add(nodeId, worker);
}
}
else
{
worker = workerProxyMap[nodeId];
}
}
existingWorkerMap[nodeId].State = WorkerState.BUSY;
worker.receiveTask(splitMetadata);
}
catch (Exception ex)
{
taskList[splitMetadata.SplitId].StatusType = StatusType.NOT_SEND_TO_WORKER;
Common.Logger().LogError("Unable to send split " + splitMetadata.SplitId + " to node " + nodeId, string.Empty, string.Empty);
Common.Logger().LogError(ex.Message, string.Empty, string.Empty);
}
}
else
{
Common.Logger().LogError("tracker with id " + workerId + " has been freezed so no new splits sent by him", string.Empty, string.Empty);
}
}
private void TimerCallback(object state)
{
int downVotesCount = 0;
IWorkerTracker worker = null;
int workerCount = existingWorkerMap.Count;
List <int> workerIdLIst = new List <int>(existingWorkerMap.Keys);
for (int i = 0; i < workerIdLIst.Count; i++)
{
if (existingWorkerMap.ContainsKey(workerIdLIst[i]) && !IsTrackerFreezed)
{
try
{
lock (workerProxyMap)
{
if (!workerProxyMap.ContainsKey(workerIdLIst[i]))
{
string url = existingWorkerMap[workerIdLIst[i]].Nodeurl;
worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url);
if (worker != null)
{
workerProxyMap.Add(workerIdLIst[i], worker);
}
}
else
{
worker = workerProxyMap[workerIdLIst[i]];
}
}
if (!worker.checkHeartbeat(myURL))
{
downVotesCount++;
}
if (downVotesCount >= existingWorkerMap.Count)
{
stopHeatBeat();
Worker.isJobTracker = false;
Console.WriteLine("I'm no longer the tracker");
}
}
catch (Exception ex)
{
if (existingWorkerMap.ContainsKey(workerIdLIst[i]))
{
existingWorkerMap.Remove(workerIdLIst[i]);
workerCount--;
/*Thread taskUpdateThread = new Thread(() => updateTaskList(workerIdLIst[i]));
* taskUpdateThread.Start();*/
updateTaskList(workerIdLIst[i]);
/*do the failed notification sending in a seperate thread*/
/*Thread thread = new Thread(() => notifyWorkersAboutFailedNode(workerIdLIst[i]));
* thread.Start();*/
notifyWorkersAboutFailedNode(workerIdLIst[i]);
Common.Logger().LogError("node " + workerIdLIst[i] + " was removed while during heartbeat", string.Empty, string.Empty);
Console.WriteLine("Removed worker from map*****************" + "node is " + workerIdLIst[i]);
}
}
}
}
}
internal Dictionary <StatusType, List <int> > getStatusForWorker(Dictionary <StatusType, List <int> > freezedWorkerStatus, int nodeId, string nodeURL)
{
//Worker has come back. first thing to do is add to worker map and notify others
WorkerDetails workerObj = getWorker(nodeId, nodeURL);
if (!existingWorkerMap.ContainsKey(nodeId))
{
existingWorkerMap.Add(nodeId, workerObj);
Thread thread = new Thread(() => notifyWorkersAboutUnfreezed(nodeId, nodeURL));
thread.Start();
}
//make him upto date
List <Int32> inProgress = new List <int>();
List <Int32> sendToClient = new List <int>();
bool isTaskAvailable = false;
Dictionary <StatusType, List <int> > result = new Dictionary <StatusType, List <int> >();
for (int i = 0; i < freezedWorkerStatus.Count; i++)
{
KeyValuePair <StatusType, List <int> > entry = freezedWorkerStatus.ElementAt(i);
switch (entry.Key)
{
case StatusType.COMPLETED:
foreach (int split in entry.Value)
{
lock (taskList[split])
{
if (taskList[split].StatusType == StatusType.INPROGRESS || taskList[split].StatusType == StatusType.NOT_SEND_TO_WORKER)
{
if (taskList[split].StatusType == StatusType.INPROGRESS)
{
string url = existingWorkerMap[taskList[split].WorkerId].Nodeurl;
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url);
worker.suspendTask(split);
}
taskList[split].WorkerId = nodeId; //later will receive completed once result sent to client
sendToClient.Add(split);
}
}
}
break;
case StatusType.NOT_STARTED:
foreach (int split in entry.Value)
{
lock (taskList[split])
{
if (taskList[split].StatusType == StatusType.INPROGRESS && taskList[split].WorkerId != nodeId && taskList[split].PercentageCompleted > Constants.jobReplaceBoundaryPercentage || taskList[split].StatusType == StatusType.COMPLETED)
{
isTaskAvailable = false;
}
else
{
inProgress.Add(split);
isTaskAvailable = true;
}
}
}
break;
default:
break;
}
}
if (isTaskAvailable == false)
{
FileSplitMetadata newSplitData = getNextPendingSplitFromList(nodeId);
if (newSplitData != null)
{
sendTaskToWorker(nodeId, newSplitData);
existingWorkerMap[nodeId].State = WorkerState.BUSY;
}
}
else
{
existingWorkerMap[nodeId].State = WorkerState.BUSY;
}
result.Add(StatusType.COMPLETED, sendToClient); //cmpare completed and ResultTask and remove from resulttask if not in
result.Add(StatusType.NOT_STARTED, inProgress); //compare with split metatdata,and remove from metadata if not
return(result);
}
internal void updateRecoveredWorker(Dictionary <StatusType, List <int> > updatedStatus, String workerURL)
{
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), workerURL);
worker.updateRecoveredWorker(updatedStatus);
}
internal Dictionary <StatusType, List <int> > getRecoveredStatus(string nodeURL)
{
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), nodeURL);
return(worker.getRecoveryStatus());
}
internal Dictionary <Int32, WorkerDetails> getExistingWorkerURLList(string entryURL)
{
IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entryURL);
return(worker.getExistingWorkers());
}
