public void WorkerFailureHandle(FaultyWorkerEventArgs args) { OverseenWorker worker = null; worker = manager.GetOverseenWorker(args.WorkerId); if (worker == null) { return; } DisposeWorker(worker.WorkerId); if (worker.WorkerId == id) { return; } // TODO: this just hides the phantom worker bug if (worker.Assignment == null) { logger.Log("WORKER FAILURE: W" + args.WorkerId); return; } logger.Log("WORKER FAILURE WITH JOBS TO DO: W" + args.WorkerId); int workerControllerId = (worker.WorkerId + 1) % (manager.Count + 1); if (workerControllerId == 0) { workerControllerId = 1; } OverseenWorker controllerWorker = manager.GetOverseenWorker(workerControllerId); logger.Log("JUST CHEKING THE CONTROLLER OF WORKER " + worker.WorkerId + " IS THE RIGHT ONE " + controllerWorker.WorkerId); if (controllerWorker != null) { while (true) { // Reassign work - search for the first worker that is free if (controllerWorker.Assignment == null) { logger.Log("Assigning work to: W" + controllerWorker.WorkerId); controllerWorker.Assignment = worker.Assignment; if (controllerWorker.Assignment != null) { controllerWorker.Assignment.WorkerId = controllerWorker.WorkerId; } // Gets the service for the worker, but checks if that worker is himself WorkerService otherWorkerService = null; if (controllerWorker.WorkerId == Id) { AsyncSetWorkerResumeContext submit = new AsyncSetWorkerResumeContext(this.LoadResumeContext); submit.BeginInvoke(clientURL, mapClass, code, controllerWorker.Assignment.Jobs, null, null); } else { otherWorkerService = controllerWorker.Service; otherWorkerService.LoadResumeContext(clientURL, this.mapClass, this.code, controllerWorker.Assignment.Jobs); } return; } // What if you don't find any free worker? // Sleep for 5 seconds and go to the beginning of the loop Thread.Sleep(5000); } } else { // Do this ad infinitum until finding a free worker (in case they are all busy atm) while (true) { // Reassign work - search for the first worker that is free foreach (OverseenWorker otherWorker in manager.ListWorkers) { if (otherWorker.Assignment == null) { logger.Log("Assigning work to: W" + otherWorker.WorkerId); otherWorker.Assignment = worker.Assignment; if (otherWorker.Assignment != null) { otherWorker.Assignment.WorkerId = otherWorker.WorkerId; } // Gets the service for the worker, but checks if that worker is himself WorkerService otherWorkerService = null; if (otherWorker.WorkerId == Id) { AsyncSetWorkerResumeContext submit = new AsyncSetWorkerResumeContext(this.LoadResumeContext); submit.BeginInvoke(clientURL, mapClass, code, otherWorker.Assignment.Jobs, null, null); } else { otherWorkerService = otherWorker.Service; otherWorkerService.LoadResumeContext(clientURL, this.mapClass, this.code, otherWorker.Assignment.Jobs); } return; } } // What if you don't find any free worker? // Sleep for 5 seconds and go to the beginning of the loop Thread.Sleep(5000); } } }
/// <summary> /// Event fired when timer expires, removing the worker /// from the available worker pool. /// Also needs to reassign the job that the worker /// was in charge of so someone else can finish it. /// </summary> /// <param name="sender"></param> /// <param name="e"></param> private void OnFaultyWorkerTimeExpiredEvent(System.Timers.Timer timer, FaultyWorkerEventArgs args) { WorkerFailureHandle(args); }
public void WorkerFailureHandle(FaultyWorkerEventArgs args) { OverseenWorker worker = null; worker = manager.GetOverseenWorker(args.WorkerId); if (worker == null) { return; } DisposeWorker(worker.WorkerId); if (worker.WorkerId == id) { return; } // TODO: this just hides the phantom worker bug if (worker.Assignment == null) { logger.Log("WORKER FAILURE: W" + args.WorkerId); return; } logger.Log("WORKER FAILURE WITH JOBS TO DO: W" + args.WorkerId); int workerControllerId = (worker.WorkerId + 1) % (manager.Count + 1); if (workerControllerId == 0) workerControllerId = 1; OverseenWorker controllerWorker = manager.GetOverseenWorker(workerControllerId); logger.Log("JUST CHEKING THE CONTROLLER OF WORKER " + worker.WorkerId + " IS THE RIGHT ONE " + controllerWorker.WorkerId); if (controllerWorker != null) { while (true) { // Reassign work - search for the first worker that is free if (controllerWorker.Assignment == null) { logger.Log("Assigning work to: W" + controllerWorker.WorkerId); controllerWorker.Assignment = worker.Assignment; if (controllerWorker.Assignment != null) { controllerWorker.Assignment.WorkerId = controllerWorker.WorkerId; } // Gets the service for the worker, but checks if that worker is himself WorkerService otherWorkerService = null; if (controllerWorker.WorkerId == Id) { AsyncSetWorkerResumeContext submit = new AsyncSetWorkerResumeContext(this.LoadResumeContext); submit.BeginInvoke(clientURL, mapClass, code, controllerWorker.Assignment.Jobs, null, null); } else { otherWorkerService = controllerWorker.Service; otherWorkerService.LoadResumeContext(clientURL, this.mapClass, this.code, controllerWorker.Assignment.Jobs); } return; } // What if you don't find any free worker? // Sleep for 5 seconds and go to the beginning of the loop Thread.Sleep(5000); } } else { // Do this ad infinitum until finding a free worker (in case they are all busy atm) while (true) { // Reassign work - search for the first worker that is free foreach (OverseenWorker otherWorker in manager.ListWorkers) { if (otherWorker.Assignment == null) { logger.Log("Assigning work to: W" + otherWorker.WorkerId); otherWorker.Assignment = worker.Assignment; if (otherWorker.Assignment != null) { otherWorker.Assignment.WorkerId = otherWorker.WorkerId; } // Gets the service for the worker, but checks if that worker is himself WorkerService otherWorkerService = null; if (otherWorker.WorkerId == Id) { AsyncSetWorkerResumeContext submit = new AsyncSetWorkerResumeContext(this.LoadResumeContext); submit.BeginInvoke(clientURL, mapClass, code, otherWorker.Assignment.Jobs, null, null); } else { otherWorkerService = otherWorker.Service; otherWorkerService.LoadResumeContext(clientURL, this.mapClass, this.code, otherWorker.Assignment.Jobs); } return; } } // What if you don't find any free worker? // Sleep for 5 seconds and go to the beginning of the loop Thread.Sleep(5000); } } }