//-------------------------------------------------------------------------------------------------//
private bool NotifyServiceBroker(ExperimentInfo experimentInfo)
{
const string STRLOG_MethodName = "NotifyServiceBroker";
Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);
bool success = false;
try
{
//
// Notify the ServiceBroker so that the results can be retrieved
//
LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.allowedServiceBrokers);
if ((success = labServerToSbAPI.Notify(experimentInfo.experimentId, experimentInfo.sbName)) == true)
{
success = this.experimentResults.UpdateNotified(experimentInfo.experimentId, experimentInfo.sbName);
}
}
catch (Exception ex)
{
Logfile.WriteError(ex.Message);
}
string logMessage = STRLOG_success + success.ToString();
Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName, logMessage);
return(success);
}
//-------------------------------------------------------------------------------------------------//
private bool NotifyServiceBroker(ExperimentInfo experimentInfo)
{
const string STRLOG_MethodName = "NotifyServiceBroker";
Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);
bool success = false;
try
{
//
// Notify the ServiceBroker so that the results can be retrieved
//
LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.allowedServiceBrokers);
if ((success = labServerToSbAPI.Notify(experimentInfo.experimentId, experimentInfo.sbName)) == true)
{
success = this.experimentResults.UpdateNotified(experimentInfo.experimentId, experimentInfo.sbName);
}
}
catch (Exception ex)
{
Logfile.WriteError(ex.Message);
}
string logMessage = STRLOG_success + success.ToString();
Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName, logMessage);
return success;
}
//-------------------------------------------------------------------------------------------------//
private void LabExperimentManagerThread()
{
const string STRLOG_MethodName = "LabExperimentManagerThread";
Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);
//
// Initialise state machine
//
this.Running = true;
this.ManagerStatus = StatusCodes.Running;
States state = States.sInit;
States lastState = States.sStart;
int lastEngineIndex = this.appData.farmSize - 1;
//
// State machine loop
//
try
{
bool success;
while (this.Running == true)
{
//
// Display message on each state change
//
if (state != lastState)
{
string logMessage = " [ " + STRLOG_MethodName + ": " + lastState.ToString() + " -> " + state.ToString() + " ]";
Logfile.Write(logMessage);
Trace.WriteLine(logMessage);
lastState = state;
}
switch (state)
{
case States.sInit:
//
// Update LabManager status
//
lock (this.statusLock)
{
this.slOnline = true;
this.slLabStatusMessage = StatusCodes.Ready.ToString();
}
//
// Revert any 'Running' experiments back to 'Waiting' so that they can be run again
//
ExperimentInfo[] allRunning = this.experimentQueue.RetrieveAllWithStatus(StatusCodes.Running);
for (int i = 0; i < allRunning.Length; i++)
{
int experimentId = allRunning[i].experimentId;
string sbName = allRunning[i].sbName;
success = this.experimentQueue.UpdateStatus(allRunning[i].experimentId, allRunning[i].sbName, StatusCodes.Waiting);
string logMessage = STRLOG_RevertingToWaiting + STRLOG_experimentId + experimentId.ToString() +
Logfile.STRLOG_Spacer + Logfile.STRLOG_Quote + STRLOG_sbName + sbName + Logfile.STRLOG_Quote +
Logfile.STRLOG_Spacer + STRLOG_success + success.ToString();
Logfile.Write(logMessage);
}
//
// Check if any experiments have not notified their ServiceBroker
//
this.submittedSinceLastNotifyCheck = true;
state = States.sIdle;
break;
case States.sIdle:
this.ManagerStatus = StatusCodes.Ready;
//
// Wait for an experiment to be submitted or timeout after QUEUE_CHECK_DELAY_SECS.
// In either case, check the experiment queue. Maybe an experiment submission
// signal got missed and it didn't get seen here. It has happened before.
//
lock (this.signalSubmitted)
{
if (Monitor.Wait(this.signalSubmitted, QUEUE_CHECK_DELAY_SECS * 1000) == true)
{
//
// Signal received, go check the queue
//
this.submittedSinceLastNotifyCheck = true;
state = States.sCheckExperimentQueue;
break;
}
}
//
// Timed out, go check some other things
//
if (this.submittedSinceLastNotifyCheck == true)
{
this.ManagerStatus = StatusCodes.Running;
state = States.sCheckNotified;
}
break;
case States.sCheckExperimentQueue:
//
// Wait a bit before checking the queue
//
Thread.Sleep(1000);
//
// Check the queue to see if there are any experiments waiting
//
if (this.experimentQueue.GetWaitCount() > 0)
{
state = States.sFindAvailableEngine;
break;
}
state = States.sIdle;
break;
case States.sFindAvailableEngine:
//
// Find an available experiment engine to run a waiting experiment
//
bool foundAvailableEngine = false;
for (int i = 0; i < this.appData.farmSize; i++)
{
//
// Determine which engine to look at
//
#if x
if (++lastEngineIndex == this.appData.farmSize)
{
lastEngineIndex = 0;
}
#else
lastEngineIndex = i;
#endif
//
// Determine if this experiment engine is currently running
//
LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[lastEngineIndex];
bool isRunning = labExperimentEngine.IsRunning;
string logMessage = " i: " + i.ToString() +
Logfile.STRLOG_Spacer + STRLOG_unitId + labExperimentEngine.UnitId.ToString() +
Logfile.STRLOG_Spacer + STRLOG_IsRunning + isRunning.ToString();
Logfile.Write(logMessage);
//
// Try starting the experiment engine, may already be running or offline
//
if (labExperimentEngine.Start() == true)
{
//
// An available engine has been found and started, check the queue again
//
foundAvailableEngine = true;
state = States.sCheckExperimentQueue;
break;
}
}
//
// Check if an available engine was found
//
if (foundAvailableEngine == false)
{
//
// Not found, have to wait for one
//
state = States.sWaitForAvailableEngine;
}
break;
case States.sWaitForAvailableEngine:
//
// Wait for an experiment engine to complete execution or timeout after ENGINE_CHECK_DELAY_SECS.
// In either case, check for an available engine. Maybe an experiment engine completion
// signal got missed and it didn't get seen here. It has happened before.
//
lock (this.signalCompleted)
{
if (Monitor.Wait(this.signalCompleted, ENGINE_CHECK_DELAY_SECS * 1000) == true)
{
//
// Signal received, find an available engine to run the next experiment
//
state = States.sFindAvailableEngine;
}
}
//
// No experiment engine has signalled its completion, check the queue again
//
state = States.sCheckExperimentQueue;
break;
case States.sCheckNotified:
//
// Check if any experiments have not notified their ServiceBroker
//
success = true;
ResultsIdInfo[] allNotNotified = this.experimentResults.RetrieveAllNotNotified();
for (int i = 0; i < allNotNotified.Length && success == true; i++)
{
int experimentId = allNotNotified[i].experimentId;
string sbName = allNotNotified[i].sbName;
try
{
//
// Attempt to notify the ServiceBroker for this experiment
//
LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.appData.allowedServiceBrokers);
if ((success = labServerToSbAPI.Notify(experimentId, sbName)) == true)
{
success = this.experimentResults.UpdateNotified(experimentId, sbName);
}
}
catch (Exception ex)
{
Logfile.WriteError(ex.Message);
success = false;
}
}
if (success == true)
{
//
// All notifies completed, don't check again until next experiment submission
//
this.submittedSinceLastNotifyCheck = false;
}
state = States.sCheckExperimentQueue;
break;
}
}
}
catch (Exception ex)
{
Logfile.WriteError(STRLOG_MethodName + ": " + ex.Message);
}
Trace.WriteLine(STRLOG_MethodName + ": Exiting");
Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName);
}
//-------------------------------------------------------------------------------------------------//
private void LabExperimentManagerThread()
{
const string STRLOG_MethodName = "LabExperimentManagerThread";
Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);
//
// Initialise state machine
//
this.Running = true;
this.ManagerStatus = StatusCodes.Running;
States state = States.sInit;
States lastState = States.sStart;
int lastEngineIndex = this.appData.farmSize - 1;
//
// State machine loop
//
try
{
bool success;
while (this.Running == true)
{
//
// Display message on each state change
//
if (state != lastState)
{
string logMessage = " [ " + STRLOG_MethodName + ": " + lastState.ToString() + " -> " + state.ToString() + " ]";
Logfile.Write(logMessage);
Trace.WriteLine(logMessage);
lastState = state;
}
switch (state)
{
case States.sInit:
//
// Update LabManager status
//
lock (this.statusLock)
{
this.slOnline = true;
this.slLabStatusMessage = StatusCodes.Ready.ToString();
}
//
// Revert any 'Running' experiments back to 'Waiting' so that they can be run again
//
ExperimentInfo[] allRunning = this.experimentQueue.RetrieveAllWithStatus(StatusCodes.Running);
for (int i = 0; i < allRunning.Length; i++)
{
int experimentId = allRunning[i].experimentId;
string sbName = allRunning[i].sbName;
success = this.experimentQueue.UpdateStatus(allRunning[i].experimentId, allRunning[i].sbName, StatusCodes.Waiting);
string logMessage = STRLOG_RevertingToWaiting + STRLOG_experimentId + experimentId.ToString() +
Logfile.STRLOG_Spacer + Logfile.STRLOG_Quote + STRLOG_sbName + sbName + Logfile.STRLOG_Quote +
Logfile.STRLOG_Spacer + STRLOG_success + success.ToString();
Logfile.Write(logMessage);
}
//
// Check if any experiments have not notified their ServiceBroker
//
this.submittedSinceLastNotifyCheck = true;
state = States.sIdle;
break;
case States.sIdle:
this.ManagerStatus = StatusCodes.Ready;
//
// Wait for an experiment to be submitted or timeout after QUEUE_CHECK_DELAY_SECS.
// In either case, check the experiment queue. Maybe an experiment submission
// signal got missed and it didn't get seen here. It has happened before.
//
lock (this.signalSubmitted)
{
if (Monitor.Wait(this.signalSubmitted, QUEUE_CHECK_DELAY_SECS * 1000) == true)
{
//
// Signal received, go check the queue
//
this.submittedSinceLastNotifyCheck = true;
state = States.sCheckExperimentQueue;
break;
}
}
//
// Timed out, go check some other things
//
if (this.submittedSinceLastNotifyCheck == true)
{
this.ManagerStatus = StatusCodes.Running;
state = States.sCheckNotified;
}
break;
case States.sCheckExperimentQueue:
//
// Wait a bit before checking the queue
//
Thread.Sleep(1000);
//
// Check the queue to see if there are any experiments waiting
//
if (this.experimentQueue.GetWaitCount() > 0)
{
state = States.sFindAvailableEngine;
break;
}
state = States.sIdle;
break;
case States.sFindAvailableEngine:
//
// Find an available experiment engine to run a waiting experiment
//
bool foundAvailableEngine = false;
for (int i = 0; i < this.appData.farmSize; i++)
{
//
// Determine which engine to look at
//
#if x
if (++lastEngineIndex == this.appData.farmSize)
{
lastEngineIndex = 0;
}
#else
lastEngineIndex = i;
#endif
//
// Determine if this experiment engine is currently running
//
LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[lastEngineIndex];
bool isRunning = labExperimentEngine.IsRunning;
string logMessage = " i: " + i.ToString() +
Logfile.STRLOG_Spacer + STRLOG_unitId + labExperimentEngine.UnitId.ToString() +
Logfile.STRLOG_Spacer + STRLOG_IsRunning + isRunning.ToString();
Logfile.Write(logMessage);
//
// Try starting the experiment engine, may already be running or offline
//
if (labExperimentEngine.Start() == true)
{
//
// An available engine has been found and started, check the queue again
//
foundAvailableEngine = true;
state = States.sCheckExperimentQueue;
break;
}
}
//
// Check if an available engine was found
//
if (foundAvailableEngine == false)
{
//
// Not found, have to wait for one
//
state = States.sWaitForAvailableEngine;
}
break;
case States.sWaitForAvailableEngine:
//
// Wait for an experiment engine to complete execution or timeout after ENGINE_CHECK_DELAY_SECS.
// In either case, check for an available engine. Maybe an experiment engine completion
// signal got missed and it didn't get seen here. It has happened before.
//
lock (this.signalCompleted)
{
if (Monitor.Wait(this.signalCompleted, ENGINE_CHECK_DELAY_SECS * 1000) == true)
{
//
// Signal received, find an available engine to run the next experiment
//
state = States.sFindAvailableEngine;
}
}
//
// No experiment engine has signalled its completion, check the queue again
//
state = States.sCheckExperimentQueue;
break;
case States.sCheckNotified:
//
// Check if any experiments have not notified their ServiceBroker
//
success = true;
ResultsIdInfo[] allNotNotified = this.experimentResults.RetrieveAllNotNotified();
for (int i = 0; i < allNotNotified.Length && success == true; i++)
{
int experimentId = allNotNotified[i].experimentId;
string sbName = allNotNotified[i].sbName;
try
{
//
// Attempt to notify the ServiceBroker for this experiment
//
LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.appData.allowedServiceBrokers);
if ((success = labServerToSbAPI.Notify(experimentId, sbName)) == true)
{
success = this.experimentResults.UpdateNotified(experimentId, sbName);
}
}
catch (Exception ex)
{
Logfile.WriteError(ex.Message);
success = false;
}
}
if (success == true)
{
//
// All notifies completed, don't check again until next experiment submission
//
this.submittedSinceLastNotifyCheck = false;
}
state = States.sCheckExperimentQueue;
break;
}
}
}
catch (Exception ex)
{
Logfile.WriteError(STRLOG_MethodName + ": " + ex.Message);
}
Trace.WriteLine(STRLOG_MethodName + ": Exiting");
Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName);
}