private void _periodicPendingJobsCheckTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) { lock (_pendingJobs) for (int i = 0; i < _pendingJobs.Count; i++) { PendingJobRecord record = _pendingJobs[i]; if ((DateTime.Now - record.StartTime) > record.MaxTimeToComplete) { // "_pendingJobs[i].Worker" is unresponsive! // We should restore it's associated jobs to the send queue // and remove this worker from _availableWorkers (if it is currently in it) // if it is not in _availableWorkers, so it is being used by a dispatcher thread, we should flag it // to be removed from _availableWorkers. // Restore jobs to the send queue AddJobs(record.Jobs.ToArray()); // Remove record _pendingJobs.Remove(record); // record.Worker.IsUnresponsive = true; // Generate UnresponsiveWorkerLog } } }
public void JobResultRecived(Quax.JobResult result) { // This method is invoked by "Quax" object whenever a result recived from a worker // "Quax" object is informed by "JobResultReciver" object // Update pending jobs table PendingJobRecord record = null; Job pendingJob = null; lock (_pendingJobs) { for (int i = 0; i < _pendingJobs.Count; i++) { if (_pendingJobs[i].Worker.ID == result.WorkerID) { for (int j = 0; j < _pendingJobs[i].Jobs.Count; j++) { if (_pendingJobs[i].Jobs[j].ID == result.JobID) { record = _pendingJobs[i]; pendingJob = _pendingJobs[i].Jobs[j]; } } } } if (pendingJob != null) { // Pending job found, we should remove it from table and update its fields record.Jobs.Remove(pendingJob); record.ExpectedTimeToComplete -= pendingJob.ExpectedTimeToComplete; record.MaxTimeToComplete -= pendingJob.MaxTimeToComplete; // If record has no associated job, we can remove it from table if (record.Jobs.Count == 0) { _pendingJobs.Remove(record); } } /* If (pendingJob == null) ===> pending job is removed in the past (2 job result * is recived for 1 unique job by to diffrent worker!) */ } }
private void JobDispatchLoop() { const int MAX_PAYLOAD_SIZE = 4 * 1024 * 1024; // 4 Megabytes bool threadShouldContinue = true; bool readyToDispatch = false; Worker selectedWorker = null; List <Job> selectedJobs = new List <Job>(16); Job[] selectedJobsArray; long selectedJobsTotalSize = 0; while (threadShouldContinue && _dispatcherThreadsLife) { readyToDispatch = false; lock (_sendQueue) { lock (_avialableWorkers) { // Remove Unresponsive workers from the _available queue bool doMore = true; while ((_avialableWorkers.Count > 0) && doMore) { if (_avialableWorkers.Peek().IsUnresponsive) { _avialableWorkers.Dequeue(); } else { doMore = false; } } // if ((_sendQueue.Count > 0) && (_avialableWorkers.Count > 0) && _isDispatching) { // Select a worker and some jobs to dispatch to it selectedWorker = _avialableWorkers.Dequeue(); _activeWorkers.Add(selectedWorker); #region Select jobs // Maximum number of jobs (per worker) = 256 // Maximum size of jobs: 16 Mega Bytes = 16*1024*1024 Bytes selectedJobs.Clear(); // Cleare jobs in the queue (from last dispatch) while ((selectedJobs.Count < _jobsPerWorker) && (_sendQueue.Count > 0)) { Job j; long jobTotalLength = 0; j = _sendQueue.Dequeue(); try { jobTotalLength = j.GetTotalLength(); } catch { // Error in reading job executable file info //JobExecutableFileReadErrorLog log; } // Check length of job if (jobTotalLength > MAX_PAYLOAD_SIZE) { // Invalid Job Log.JobTooBigLog log = new Log.JobTooBigLog(); log.Message = "Total length of executable file and input file of the job is greater than 4 MB, this job will not send to workers."; log.Job = j; NewLogItem(log); } else if ((selectedJobsTotalSize + jobTotalLength) <= MAX_PAYLOAD_SIZE) { // Add job selectedJobsTotalSize += jobTotalLength; selectedJobs.Add(j); } else { // There is no space for other jobs, restore job to the send queue and break loop _sendQueue.Enqueue(j); break; } } if (selectedJobs.Count > 0) { readyToDispatch = true; } #endregion #region Control number of dispatcher threads if (_numberOfDispatcherThreads > _avialableWorkers.Count && _numberOfDispatcherThreads > 1) { // This Thread should terminate after this iteration (after sending the job) threadShouldContinue = false; _numberOfDispatcherThreads--; } else if (_numberOfDispatcherThreads < _avialableWorkers.Count && _numberOfDispatcherThreads < MaxDispatcherThreads) { // Create a new dispatcher thread CreateNewDispatcherThread(); } #endregion } // end of if } // end of lock(_availableWorkers) if (!readyToDispatch && threadShouldContinue) { Monitor.Wait(_sendQueue, 2000); } } // end of lock(_sendQueue) if (readyToDispatch) { #region Start dispatching jobs bool successfullySent; bool networkSendError; //Start dispatching selected jobs to worker "selectedWorker" selectedJobsArray = selectedJobs.ToArray(); successfullySent = DispatchJobs(selectedJobsArray, selectedWorker, (uint)selectedJobsTotalSize, out networkSendError); // Restore "selectedWorker" to available worekrs ((if it was working properly)) _activeWorkers.Remove(selectedWorker); // "selectedWorker" is not active any more if (successfullySent || (!successfullySent && !networkSendError)) { lock (_avialableWorkers) _avialableWorkers.Enqueue(selectedWorker); } if (successfullySent) { #region Create or update pending job record PendingJobRecord record = null; lock (_pendingJobs) { // Try to find a pending job record for (int i = 0; i < _pendingJobs.Count; i++) { if (_pendingJobs[i].Worker.ID == selectedWorker.ID) { record = _pendingJobs[i]; break; } } // Record not found, we must create one new recorde if (record == null) { record = new PendingJobRecord(); record.Worker = selectedWorker; record.StartTime = DateTime.Now; _pendingJobs.Add(record); } // Update the record //TimeSpan expectedTimeIncrease = TimeSpan.Zero; //TimeSpan MaxTimeIncrease = TimeSpan.Zero; for (int i = 0; i < selectedJobsArray.Length; i++) { record.Jobs.Add(selectedJobsArray[i]); record.ExpectedTimeToComplete = record.ExpectedTimeToComplete.Add(selectedJobsArray[i].ExpectedTimeToComplete); record.MaxTimeToComplete = record.MaxTimeToComplete.Add(selectedJobsArray[i].MaxTimeToComplete); } } #endregion } else { // Send failed, we should restore jobs to send queue AddJobs(selectedJobsArray); } #endregion } } // End of while }