private void _periodicPendingJobsCheckTimer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
{
lock (_pendingJobs)
for (int i = 0; i < _pendingJobs.Count; i++)
{
PendingJobRecord record = _pendingJobs[i];
if ((DateTime.Now - record.StartTime) > record.MaxTimeToComplete)
{
// "_pendingJobs[i].Worker" is unresponsive!
// We should restore it's associated jobs to the send queue
// and remove this worker from _availableWorkers (if it is currently in it)
// if it is not in _availableWorkers, so it is being used by a dispatcher thread, we should flag it
// to be removed from _availableWorkers.
// Restore jobs to the send queue
AddJobs(record.Jobs.ToArray());
// Remove record
_pendingJobs.Remove(record);
//
record.Worker.IsUnresponsive = true;
// Generate UnresponsiveWorkerLog
}
}
}
public void JobResultRecived(Quax.JobResult result)
{
// This method is invoked by "Quax" object whenever a result recived from a worker
// "Quax" object is informed by "JobResultReciver" object
// Update pending jobs table
PendingJobRecord record = null;
Job pendingJob = null;
lock (_pendingJobs) {
for (int i = 0; i < _pendingJobs.Count; i++)
{
if (_pendingJobs[i].Worker.ID == result.WorkerID)
{
for (int j = 0; j < _pendingJobs[i].Jobs.Count; j++)
{
if (_pendingJobs[i].Jobs[j].ID == result.JobID)
{
record = _pendingJobs[i];
pendingJob = _pendingJobs[i].Jobs[j];
}
}
}
}
if (pendingJob != null)
{
// Pending job found, we should remove it from table and update its fields
record.Jobs.Remove(pendingJob);
record.ExpectedTimeToComplete -= pendingJob.ExpectedTimeToComplete;
record.MaxTimeToComplete -= pendingJob.MaxTimeToComplete;
// If record has no associated job, we can remove it from table
if (record.Jobs.Count == 0)
{
_pendingJobs.Remove(record);
}
}
/* If (pendingJob == null) ===> pending job is removed in the past (2 job result
* is recived for 1 unique job by to diffrent worker!) */
}
}
private void JobDispatchLoop()
{
const int MAX_PAYLOAD_SIZE = 4 * 1024 * 1024; // 4 Megabytes
bool threadShouldContinue = true;
bool readyToDispatch = false;
Worker selectedWorker = null;
List <Job> selectedJobs = new List <Job>(16);
Job[] selectedJobsArray;
long selectedJobsTotalSize = 0;
while (threadShouldContinue && _dispatcherThreadsLife)
{
readyToDispatch = false;
lock (_sendQueue) {
lock (_avialableWorkers) {
// Remove Unresponsive workers from the _available queue
bool doMore = true;
while ((_avialableWorkers.Count > 0) && doMore)
{
if (_avialableWorkers.Peek().IsUnresponsive)
{
_avialableWorkers.Dequeue();
}
else
{
doMore = false;
}
}
//
if ((_sendQueue.Count > 0) && (_avialableWorkers.Count > 0) && _isDispatching)
{
// Select a worker and some jobs to dispatch to it
selectedWorker = _avialableWorkers.Dequeue();
_activeWorkers.Add(selectedWorker);
#region Select jobs
// Maximum number of jobs (per worker) = 256
// Maximum size of jobs: 16 Mega Bytes = 16*1024*1024 Bytes
selectedJobs.Clear(); // Cleare jobs in the queue (from last dispatch)
while ((selectedJobs.Count < _jobsPerWorker) && (_sendQueue.Count > 0))
{
Job j;
long jobTotalLength = 0;
j = _sendQueue.Dequeue();
try {
jobTotalLength = j.GetTotalLength();
}
catch {
// Error in reading job executable file info
//JobExecutableFileReadErrorLog log;
}
// Check length of job
if (jobTotalLength > MAX_PAYLOAD_SIZE)
{
// Invalid Job
Log.JobTooBigLog log = new Log.JobTooBigLog();
log.Message = "Total length of executable file and input file of the job is greater than 4 MB, this job will not send to workers.";
log.Job = j;
NewLogItem(log);
}
else if ((selectedJobsTotalSize + jobTotalLength) <= MAX_PAYLOAD_SIZE)
{
// Add job
selectedJobsTotalSize += jobTotalLength;
selectedJobs.Add(j);
}
else
{
// There is no space for other jobs, restore job to the send queue and break loop
_sendQueue.Enqueue(j);
break;
}
}
if (selectedJobs.Count > 0)
{
readyToDispatch = true;
}
#endregion
#region Control number of dispatcher threads
if (_numberOfDispatcherThreads > _avialableWorkers.Count && _numberOfDispatcherThreads > 1)
{
// This Thread should terminate after this iteration (after sending the job)
threadShouldContinue = false;
_numberOfDispatcherThreads--;
}
else if (_numberOfDispatcherThreads < _avialableWorkers.Count && _numberOfDispatcherThreads < MaxDispatcherThreads)
{
// Create a new dispatcher thread
CreateNewDispatcherThread();
}
#endregion
} // end of if
} // end of lock(_availableWorkers)
if (!readyToDispatch && threadShouldContinue)
{
Monitor.Wait(_sendQueue, 2000);
}
} // end of lock(_sendQueue)
if (readyToDispatch)
{
#region Start dispatching jobs
bool successfullySent;
bool networkSendError;
//Start dispatching selected jobs to worker "selectedWorker"
selectedJobsArray = selectedJobs.ToArray();
successfullySent = DispatchJobs(selectedJobsArray, selectedWorker, (uint)selectedJobsTotalSize, out networkSendError);
// Restore "selectedWorker" to available worekrs ((if it was working properly))
_activeWorkers.Remove(selectedWorker); // "selectedWorker" is not active any more
if (successfullySent || (!successfullySent && !networkSendError))
{
lock (_avialableWorkers)
_avialableWorkers.Enqueue(selectedWorker);
}
if (successfullySent)
{
#region Create or update pending job record
PendingJobRecord record = null;
lock (_pendingJobs) {
// Try to find a pending job record
for (int i = 0; i < _pendingJobs.Count; i++)
{
if (_pendingJobs[i].Worker.ID == selectedWorker.ID)
{
record = _pendingJobs[i];
break;
}
}
// Record not found, we must create one new recorde
if (record == null)
{
record = new PendingJobRecord();
record.Worker = selectedWorker;
record.StartTime = DateTime.Now;
_pendingJobs.Add(record);
}
// Update the record
//TimeSpan expectedTimeIncrease = TimeSpan.Zero;
//TimeSpan MaxTimeIncrease = TimeSpan.Zero;
for (int i = 0; i < selectedJobsArray.Length; i++)
{
record.Jobs.Add(selectedJobsArray[i]);
record.ExpectedTimeToComplete = record.ExpectedTimeToComplete.Add(selectedJobsArray[i].ExpectedTimeToComplete);
record.MaxTimeToComplete = record.MaxTimeToComplete.Add(selectedJobsArray[i].MaxTimeToComplete);
}
}
#endregion
}
else
{
// Send failed, we should restore jobs to send queue
AddJobs(selectedJobsArray);
}
#endregion
}
} // End of while
}
