private void notifyWorkersAboutFailedNode(int failedNodeId) { IWorkerTracker worker = null; for (int i = 0; i < existingWorkerMap.Count; i++) { KeyValuePair <Int32, WorkerDetails> entry = existingWorkerMap.ElementAt(i); try { lock (workerProxyMap) { if (!workerProxyMap.ContainsKey(entry.Key)) { string url = entry.Value.Nodeurl; worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url); if (worker != null) { workerProxyMap.Add(entry.Key, worker); } } else { worker = workerProxyMap[entry.Key]; } } worker.removeFailedNode(failedNodeId); } catch (Exception ex) { Common.Logger().LogError("job failure notificatin of node ", string.Empty, string.Empty); } } }
private void notifyWorkersAboutUnfreezed(int nodeId, String nodeURL) { IWorkerTracker worker = null; for (int i = 0; i < existingWorkerMap.Count; i++) { KeyValuePair <Int32, WorkerDetails> entry = existingWorkerMap.ElementAt(i); try { lock (workerProxyMap) { if (!workerProxyMap.ContainsKey(entry.Key)) { string url = entry.Value.Nodeurl; worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url); if (worker != null) { workerProxyMap.Add(entry.Key, worker); } } else { worker = workerProxyMap[entry.Key]; } } worker.addUnfreezedNode(nodeId, nodeURL); } catch (Exception ex) { Common.Logger().LogError("Unable to notify others about unfreeze by Job Tracker", string.Empty, string.Empty); } } }
internal void notifyWorkersForJobStart() { IWorkerTracker worker = null; for (int i = 0; i < existingWorkerMap.Count; i++) { KeyValuePair <Int32, WorkerDetails> entry = existingWorkerMap.ElementAt(i); try { lock (workerProxyMap) { if (!workerProxyMap.ContainsKey(entry.Key)) { string url = entry.Value.Nodeurl; worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url); } else { worker = workerProxyMap[entry.Key]; } } worker.receiveNewJob(); } catch (Exception ex) { Common.Logger().LogError("Unable to notify others about unfreeze by Job Tracker", string.Empty, string.Empty); } } }
internal void notifyExistingWorkers(int workerId, String newWorkerURL, Dictionary <Int32, WorkerDetails> existingWorkerList) { foreach (KeyValuePair <Int32, WorkerDetails> entry in existingWorkerList) { IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entry.Value.Nodeurl); worker.addNewWorker(workerId, newWorkerURL); } }
public void notifyBackupJobtrackerUrl(string url, Dictionary <Int32, WorkerDetails> existingWorkerList) { foreach (KeyValuePair <Int32, WorkerDetails> entry in existingWorkerList) { IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entry.Value.Nodeurl); worker.SetBackupJobTrackerUrl(url); } }
public void SendTaskCopyToBackupTracker(string url, Dictionary <int, Task> taskList, string clientUrl) { IWorkerTracker bkpTracker = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), url); bkpTracker.SetCopyOfTasks(taskList, clientUrl); }
public void TrackerStabilized(Dictionary <Int32, WorkerDetails> existingWorkerList) { foreach (KeyValuePair <Int32, WorkerDetails> entry in existingWorkerList) { IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entry.Value.Nodeurl); worker.TrackerStabilized(); } }
internal void notifyRecoveryNodeToTracker(int workerID, string nodeURL, List <int> processingSplits, List <int> alreadySentSplits) { if (trackerProxy == null) { trackerProxy = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), Worker.JOBTRACKER_URL); } trackerProxy.notifyWorkerRecovery(workerID, nodeURL, processingSplits, alreadySentSplits); }
public void FeedNewTracker(int workerID, String workerURL, List <int> processingSplits, List <int> alreadySentSplits) { if (!WorkerTask.IS_WORKER_FREEZED) { trackerProxy = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), Worker.BKP_JOBTRACKER_URL); trackerProxy.ChangeTracker(workerID, workerURL, processingSplits, alreadySentSplits); } }
internal Dictionary <StatusType, List <int> > notifyTrackerOnUnfreeze(Dictionary <StatusType, List <int> > freezedWorkerStatus, int nodeId, string nodeURL) { if (trackerProxy == null) { trackerProxy = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), Worker.JOBTRACKER_URL); } Dictionary <StatusType, List <int> > updatedStatus = trackerProxy.receiveFreezedWorkerStatus(freezedWorkerStatus, nodeId, nodeURL); return(updatedStatus); }
internal void hasThresholdReached(int nodeId) { if (!WorkerTask.IS_WORKER_FREEZED) { if (trackerProxy == null) { trackerProxy = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), Worker.JOBTRACKER_URL); } Common.Logger().LogInfo("requesting task in threashold reached", string.Empty, string.Empty); trackerProxy.readyForNewTask(nodeId); } }
public void sendStatusUpdatesToTracker(Status status, WorkerTask workerTask) { try { workerTask.checkWorkerFreezed(); CheckSystemStability(); if (trackerProxy == null) { trackerProxy = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), Worker.JOBTRACKER_URL); } trackerProxy.receiveStatus(status); } catch (Exception ex) { Common.Logger().LogError("unable to send status updates by " + status.NodeId + " for percentage " + status.PercentageCompleted, string.Empty, string.Empty); } }
internal void notifyTaskCompletedEvent(int workerId, int splitId) { try { CheckSystemStability(); Common.Logger().LogInfo("Notifying task completed event for split " + splitId, string.Empty, string.Empty); if (trackerProxy == null) { trackerProxy = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), Worker.JOBTRACKER_URL); } trackerProxy.taskCompleted(workerId, splitId, Worker.serviceUrl); } catch (Exception ex) { Common.Logger().LogInfo("exception thrown while sending copleted event ", string.Empty, string.Empty); } }
public void submitTask(String entryUrl, string inputFile, string outputDir, int splits, string mapperFunctionName, string dllPath) { hasJobCompleted = false; startTime = DateTime.Now; this.inputFilePath = inputFile; this.outputDir = outputDir; this.mapperName = mapperFunctionName; this.dllPath = dllPath; // FileInfo f = new FileInfo(inputFilePath); // long s1 = f.Length; //JobMetadata jobDetails = new JobMetadata(s1, splits, url); byte[] input = File.ReadAllBytes(inputFilePath); JobMetadata jobDetails = new JobMetadata(input.Length, splits, url); contactingWorker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entryUrl); contactingWorker.receiveJobRequest(jobDetails);//calling the job tracker }
/*important event so if not send add to list back*/ internal void notifyResultsSentToClientEvent(int workerId, TaskResult taskResult, WorkerTask workerTask) { try { workerTask.checkWorkerFreezed(); CheckSystemStability(); if (trackerProxy == null) { trackerProxy = (IWorkerTracker)Activator.GetObject( typeof(IWorkerTracker), Worker.JOBTRACKER_URL); } Common.Logger().LogInfo("Sent to taskCompleted by Worker ID = " + workerId + " Split Id= " + taskResult.SplitId, string.Empty, string.Empty); trackerProxy.taskCompleted(workerId, taskResult.SplitId, Worker.serviceUrl); } catch (Exception ex) { workerTask.addTaskToTaskResults(taskResult); Common.Logger().LogInfo("exception thrown while sending copleted event ", string.Empty, string.Empty); Common.Logger().LogInfo(ex.Message, ex.StackTrace, string.Empty); } }
/*going to replace one slow task*/ private void ReplaceSlowTasks(int nodeId) { if (!HasPendingTasks()) { WorkerDetails bestWorker = null; FileSplitMetadata splitMetadata = null; int splitId = 0; lock (taskList) { foreach (var pair in taskList) { /* later change this to have from suspended as well */ if (pair.Value.StatusType == StatusType.INPROGRESS) { splitId = pair.Key; bestWorker = GetBestWorker(pair.Value, nodeId); if (bestWorker != null) { bestWorker.State = WorkerState.ABOUT_TO_BUSY; string url = existingWorkerMap[taskList[splitId].WorkerId].Nodeurl; IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url); worker.suspendTask(splitId); //reschedule splitMetadata = pair.Value.SplitMetadata; taskList[splitId].StatusType = StatusType.INPROGRESS; taskList[splitId].WorkerId = bestWorker.Nodeid; sendTaskToWorker(bestWorker.Nodeid, splitMetadata); } } } } } else { readyForNewTask(nodeId); } }
private void sendTaskToWorker(int nodeId, FileSplitMetadata splitMetadata) { if (!isTrackerFreezed) { IWorkerTracker worker = null; try { lock (workerProxyMap) { if (!workerProxyMap.ContainsKey(nodeId)) { string url = existingWorkerMap[nodeId].Nodeurl; worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url); if (worker != null) { workerProxyMap.Add(nodeId, worker); } } else { worker = workerProxyMap[nodeId]; } } existingWorkerMap[nodeId].State = WorkerState.BUSY; worker.receiveTask(splitMetadata); } catch (Exception ex) { taskList[splitMetadata.SplitId].StatusType = StatusType.NOT_SEND_TO_WORKER; Common.Logger().LogError("Unable to send split " + splitMetadata.SplitId + " to node " + nodeId, string.Empty, string.Empty); Common.Logger().LogError(ex.Message, string.Empty, string.Empty); } } else { Common.Logger().LogError("tracker with id " + workerId + " has been freezed so no new splits sent by him", string.Empty, string.Empty); } }
private void TimerCallback(object state) { int downVotesCount = 0; IWorkerTracker worker = null; int workerCount = existingWorkerMap.Count; List <int> workerIdLIst = new List <int>(existingWorkerMap.Keys); for (int i = 0; i < workerIdLIst.Count; i++) { if (existingWorkerMap.ContainsKey(workerIdLIst[i]) && !IsTrackerFreezed) { try { lock (workerProxyMap) { if (!workerProxyMap.ContainsKey(workerIdLIst[i])) { string url = existingWorkerMap[workerIdLIst[i]].Nodeurl; worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url); if (worker != null) { workerProxyMap.Add(workerIdLIst[i], worker); } } else { worker = workerProxyMap[workerIdLIst[i]]; } } if (!worker.checkHeartbeat(myURL)) { downVotesCount++; } if (downVotesCount >= existingWorkerMap.Count) { stopHeatBeat(); Worker.isJobTracker = false; Console.WriteLine("I'm no longer the tracker"); } } catch (Exception ex) { if (existingWorkerMap.ContainsKey(workerIdLIst[i])) { existingWorkerMap.Remove(workerIdLIst[i]); workerCount--; /*Thread taskUpdateThread = new Thread(() => updateTaskList(workerIdLIst[i])); * taskUpdateThread.Start();*/ updateTaskList(workerIdLIst[i]); /*do the failed notification sending in a seperate thread*/ /*Thread thread = new Thread(() => notifyWorkersAboutFailedNode(workerIdLIst[i])); * thread.Start();*/ notifyWorkersAboutFailedNode(workerIdLIst[i]); Common.Logger().LogError("node " + workerIdLIst[i] + " was removed while during heartbeat", string.Empty, string.Empty); Console.WriteLine("Removed worker from map*****************" + "node is " + workerIdLIst[i]); } } } } }
internal Dictionary <StatusType, List <int> > getStatusForWorker(Dictionary <StatusType, List <int> > freezedWorkerStatus, int nodeId, string nodeURL) { //Worker has come back. first thing to do is add to worker map and notify others WorkerDetails workerObj = getWorker(nodeId, nodeURL); if (!existingWorkerMap.ContainsKey(nodeId)) { existingWorkerMap.Add(nodeId, workerObj); Thread thread = new Thread(() => notifyWorkersAboutUnfreezed(nodeId, nodeURL)); thread.Start(); } //make him upto date List <Int32> inProgress = new List <int>(); List <Int32> sendToClient = new List <int>(); bool isTaskAvailable = false; Dictionary <StatusType, List <int> > result = new Dictionary <StatusType, List <int> >(); for (int i = 0; i < freezedWorkerStatus.Count; i++) { KeyValuePair <StatusType, List <int> > entry = freezedWorkerStatus.ElementAt(i); switch (entry.Key) { case StatusType.COMPLETED: foreach (int split in entry.Value) { lock (taskList[split]) { if (taskList[split].StatusType == StatusType.INPROGRESS || taskList[split].StatusType == StatusType.NOT_SEND_TO_WORKER) { if (taskList[split].StatusType == StatusType.INPROGRESS) { string url = existingWorkerMap[taskList[split].WorkerId].Nodeurl; IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), url); worker.suspendTask(split); } taskList[split].WorkerId = nodeId; //later will receive completed once result sent to client sendToClient.Add(split); } } } break; case StatusType.NOT_STARTED: foreach (int split in entry.Value) { lock (taskList[split]) { if (taskList[split].StatusType == StatusType.INPROGRESS && taskList[split].WorkerId != nodeId && taskList[split].PercentageCompleted > Constants.jobReplaceBoundaryPercentage || taskList[split].StatusType == StatusType.COMPLETED) { isTaskAvailable = false; } else { inProgress.Add(split); isTaskAvailable = true; } } } break; default: break; } } if (isTaskAvailable == false) { FileSplitMetadata newSplitData = getNextPendingSplitFromList(nodeId); if (newSplitData != null) { sendTaskToWorker(nodeId, newSplitData); existingWorkerMap[nodeId].State = WorkerState.BUSY; } } else { existingWorkerMap[nodeId].State = WorkerState.BUSY; } result.Add(StatusType.COMPLETED, sendToClient); //cmpare completed and ResultTask and remove from resulttask if not in result.Add(StatusType.NOT_STARTED, inProgress); //compare with split metatdata,and remove from metadata if not return(result); }
internal void updateRecoveredWorker(Dictionary <StatusType, List <int> > updatedStatus, String workerURL) { IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), workerURL); worker.updateRecoveredWorker(updatedStatus); }
internal Dictionary <StatusType, List <int> > getRecoveredStatus(string nodeURL) { IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), nodeURL); return(worker.getRecoveryStatus()); }
internal Dictionary <Int32, WorkerDetails> getExistingWorkerURLList(string entryURL) { IWorkerTracker worker = (IWorkerTracker)Activator.GetObject(typeof(IWorkerTracker), entryURL); return(worker.getExistingWorkers()); }