/// <summary>
/// Launch Master timer method in the controlThread.
/// </summary>
/// <param name="stepExectuionObject"></param>
private void MasterTimerMethod(object stepExectuionObject)
{
StepExecution stepExecution = (StepExecution)stepExectuionObject;
Logger.Debug("-------------------------------------------- Master Timer --------------------------------------------");
string stepName = stepExecution.StepName;
ControlQueue masterLifeLineQueue = stepExecution.remoteChunking._masterLifeLineQueue;
string masterNoAliveMessage = stepName + dot + bool.FalseString;
// stop timer when batch failed
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
masterLifeLineQueue.Send(masterNoAliveMessage);
return;
}
else // continue update workerMap in the stepExecution and check workers alive
{
Dictionary <string, bool> workerMap = stepExecution.remoteChunking._workerMap;
List <string> workerIDList = stepExecution.remoteChunking._workerStartedQueue.GetWorkerIDByMasterName(stepExecution.StepName);
foreach (string workerID in workerIDList)
{
if (!workerMap.ContainsKey(workerID))
{
workerMap[workerID] = true;
}
}
ControlQueue workerLifeLineQueue = stepExecution.remoteChunking._workerLifeLineQueue;
List <string> workerIDs = new List <string>(workerMap.Keys);
workerLifeLineQueue.CheckMessageExistAndConsumeAll(workerIDs, workerMap);
}
}
/// <summary>
/// Launch Worker method in the controlThread.
/// </summary>
/// <param name="stepExecution"></param>
/// <param name="threadWait"></param>
public void Worker(StepExecution stepExecution, AutoResetEvent threadWait)
{
// workerStartedMessage includes stepname and worker id
string workerStartedMessage = stepExecution.StepName + dot + stepExecution.remoteChunking.WorkerID.ToString();
// worker send workerStartedMessage in the workerStarted queue for master job to check
stepExecution.remoteChunking._workerStartedQueue.Send(workerStartedMessage);
int maxMasterWaitWorkerSecond = stepExecution.remoteChunking.MaxMasterWaitWorkerSecond;
int _workerTimerseconds = maxMasterWaitWorkerSecond * 1000;
// start a timer to let master check worker is still alive or not in every defualt seconds
Timer timer = new Timer(WorkerTimerMethod, stepExecution, 0, _workerTimerseconds);
bool Isterminate = false;
while (!Isterminate)
{
// send back signal to the afterStep and stop timer when batch failed
ControlQueue workerCompletedQueue = stepExecution.remoteChunking._workerCompletedQueue;
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
timer.Dispose();
Isterminate = true;
}
// when batch completed send back signal to the afterStep and stop timer after send completedmessage to message queue
else if ("COMPLETED".Equals(stepExecution.ExitStatus.ExitCode))
{
string workerCompletedMessage = stepExecution.JobExecution.JobInstance.JobName + dot + stepExecution.remoteChunking.WorkerID + dot + "COMPLETED";
workerCompletedQueue.Send(workerCompletedMessage);
timer.Dispose();
threadWait.Set();
Isterminate = true;
}
}
}
/// <summary>
/// Launch Worker timer method in the controlThread.
/// </summary>
/// <param name="stepExectuionObject"></param>
private void WorkerTimerMethod(object stepExectuionObject)
{
StepExecution stepExecution = (StepExecution)stepExectuionObject;
string workerIdMessage = stepExecution.StepName + dot + stepExecution.remoteChunking.WorkerID.ToString();
Logger.Info("-------------------------------------------- Worker Timer --------------------------------------------");
ControlQueue workerLifeLineQueue = stepExecution.remoteChunking._workerLifeLineQueue;
string workerAliveMessage = workerIdMessage + dot + bool.TrueString + dot + DateTime.Now.ToString();
string workerNoAliveMessage = workerIdMessage + dot + bool.FalseString + dot + DateTime.Now.ToString();
// stop timer when batch failed
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
workerLifeLineQueue.Send(workerNoAliveMessage);
Logger.Debug(workerIdMessage + "-----------NoAlive------------------");
return;
}
else // continue send workerAliveMessage to the workerLifeLineQueue
{
Logger.Debug(workerIdMessage + "-----------Alive------------------");
workerLifeLineQueue.Send(workerAliveMessage);
}
}
public static JobExecution WorkerStart(string xmlJobFile, UnityLoader loader, string hostName, string username = "******", string password = "******", int workerUpdateTimeInterval = 15)
{
ControlQueue _controlQueue = GetControlQueue(controlQueueName, hostName, username, password);
int messageCount = _controlQueue.GetMessageCount();
XmlJob job = null;
TimeSpan WorkerUpdatetimeInterval = TimeSpan.FromSeconds(workerUpdateTimeInterval);
if (messageCount != 0)
{
for (int i = 0; i < messageCount; i++)
{
string fileName = Path.GetFileName(xmlJobFile);
string message = _controlQueue.Receive(fileName);
if (ValidateMessage(message)) // for 3 items
{
Tuple <XmlJob, string, int, bool> tuple = ValidateFileName(message, xmlJobFile);
if (tuple.Item4)
{
// Resend the message to the controlQueue
if (tuple.Item3 > 0)
{
_controlQueue.Send(tuple.Item2);
}
job = tuple.Item1;
Guid guid = Guid.NewGuid();
foreach (XmlStep step in job.JobElements)
{
step.RemoteChunking = new XmlRemoteChunking();
step.RemoteChunking.HostName = hostName;
step.RemoteChunking.Master = false;
step.RemoteChunking.WorkerID = guid.ToString();
}
break;
}
}
}
}
else
{
do
{
messageCount = _controlQueue.GetMessageCount();
if (messageCount != 0)
{
for (int i = 0; i < messageCount; i++)
{
string fileName = Path.GetFileName(xmlJobFile);
string message = _controlQueue.Receive(fileName);
if (ValidateMessage(message)) // for 3 items
{
Tuple <XmlJob, string, int, bool> tuple = ValidateFileName(message, xmlJobFile);
if (tuple.Item4)
{
// Resend the message to the controlQueue
if (tuple.Item3 > 0)
{
_controlQueue.Send(tuple.Item2);
}
job = tuple.Item1;
Guid guid = Guid.NewGuid();
foreach (XmlStep step in job.JobElements)
{
step.RemoteChunking = new XmlRemoteChunking();
step.RemoteChunking.HostName = hostName;
step.RemoteChunking.Master = false;
step.RemoteChunking.WorkerID = guid.ToString();
}
break;
}
}
}
break;
}
else
{
Logger.Info("No master job provided. Wait for worker {0} seconds.", WorkerUpdatetimeInterval.TotalSeconds);
Thread.Sleep(WorkerUpdatetimeInterval);
//throw new JobExecutionException("No master job provided");
}
} while (messageCount == 0);
}
_controlQueue.Requeue();
loader.Job = job;
var jobOperator = (SimpleJobOperator)BatchRuntime.GetJobOperator(loader);
var executionId = jobOperator.StartNextInstance(job.Id);
return(jobOperator.JobExplorer.GetJobExecution((long)executionId));
}
/// <summary>
/// Launch Master method in the controlThread.
/// </summary>
/// <param name="stepExecution"></param>
/// <param name="threadWait"></param>
public void Master(StepExecution stepExecution, AutoResetEvent threadWait)
{
// clean up all message queues
stepExecution.remoteChunking.CleanAllQueue();
// get workerxml name
string WorkerXml = stepExecution.remoteChunking.WorkerFileName;
// get worker max numbers
int workerMaxNumber = stepExecution.remoteChunking.WorkerMaxNumber;
// configuration information includes stepname , workerxml name, and worker max numbers
string message = stepExecution.StepName + semicolon + WorkerXml + semicolon + workerMaxNumber.ToString();
// master send configuration information in the control queue for worker job to execute
stepExecution.remoteChunking._controlQueue.Send(message);
int maxMasterWaitWorkerSecond = stepExecution.remoteChunking.MaxMasterWaitWorkerSecond;
int maxMasterWaitWorkerRetry = stepExecution.remoteChunking.MaxMasterWaitWorkerRetry;
TimeSpan _MasterWaitWorkerTimeout = TimeSpan.FromSeconds(maxMasterWaitWorkerSecond);
// check at least one worker started
if (WaitForAtLeastWorkerStarted(stepExecution, maxMasterWaitWorkerRetry, _MasterWaitWorkerTimeout))
{
// send back signal to the beforeStep
threadWait.Set();
bool Isterminate = false;
int _masterTimerseconds = maxMasterWaitWorkerSecond * 1000;
// start a timer to check worker is still alive or not in every defualt seconds
Timer timer = new Timer(MasterTimerMethod, stepExecution, 0, _masterTimerseconds);
while (!Isterminate)
{
ControlQueue masterQueue = stepExecution.remoteChunking._masterQueue;
// send back signal to the afterStep and stop timer when batch failed
if ("FAILED".Equals(stepExecution.ExitStatus.ExitCode))
{
Isterminate = true;
timer.Dispose();
threadWait.Set();
}
// when batch completed send back signal to the afterStep and stop timer after send completedmessage to message queue and check all workers completed
else if ("COMPLETED".Equals(stepExecution.ExitStatus.ExitCode))
{
string masterCompletedMessage = "master" + dot + stepExecution.StepName + dot + stepExecution.ExitStatus.ExitCode;
int maxWorkerCompleteMessageNeeded = workerMaxNumber;
while (maxWorkerCompleteMessageNeeded > 0)
{
masterQueue.Send(masterCompletedMessage);
maxWorkerCompleteMessageNeeded--;
}
List <string> failList = WaitForWorkerCompleted(stepExecution, workerMaxNumber, maxMasterWaitWorkerSecond, _MasterWaitWorkerTimeout);
timer.Dispose();
if (failList.Count > 0)
{
stepExecution.ExitStatus = ExitStatus.Failed;
}
threadWait.Set();
Isterminate = true;
}
}
}
}
