//-------------------------------------------------------------------------------------------------//
private int GetMinRemainingRuntime()
{
const string STRLOG_MethodName = "GetLabExperimentStatus";
Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);
int minRemainingRuntime = Int32.MaxValue;
//
// Check experiment staus of each experiment engine
//
for (int i = 0; i < this.appData.farmSize; i++)
{
LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[i];
//
// Get the remaining runtime for this experiment engine
//
int remainingRuntime = labExperimentEngine.GetRemainingRuntime();
// Check if this is a smaller value
if (remainingRuntime < minRemainingRuntime)
{
minRemainingRuntime = remainingRuntime;
}
}
Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName);
return(minRemainingRuntime);
}
//-------------------------------------------------------------------------------------------------//
private LabExperimentStatus GetLabExperimentStatus(int experimentId, string sbName)
{
const string STRLOG_MethodName = "GetLabExperimentStatus";
Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);
LabExperimentStatus labExperimentStatus = null;
//
// Check experiment status of each experiment engine
//
for (int i = 0; i < this.appData.farmSize; i++)
{
LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[i];
labExperimentStatus = labExperimentEngine.GetLabExperimentStatus(experimentId, sbName);
if ((StatusCodes)labExperimentStatus.statusReport.statusCode != StatusCodes.Unknown)
{
// This engine is running the experiment
break;
}
}
Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName);
return(labExperimentStatus);
}
//-------------------------------------------------------------------------------------------------//
public bool Cancel(int experimentID, string sbName)
{
bool cancelled = false;
lock (this.managerLock)
{
//
// Try cancelling the experiment on the queue
//
cancelled = this.experimentQueue.Cancel(experimentID, sbName);
if (cancelled == true)
{
Logfile.Write(STRLOG_QueuedExperimentCancelled);
}
else
{
//
// Experiment may be currently running, try cancelling it there
//
for (int i = 0; i < this.appData.farmSize; i++)
{
LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[i];
cancelled = labExperimentEngine.Cancel(experimentID, sbName);
if (cancelled == true)
{
Logfile.Write(STRLOG_RunningExperimentCancelled);
break;
}
}
}
if (cancelled == false)
{
Logfile.Write(STRLOG_ExperimentNotCancelled);
}
}
return(cancelled);
}
//-------------------------------------------------------------------------------------------------//
private void LabExperimentManagerThread()
{
const string STRLOG_MethodName = "LabExperimentManagerThread";
Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);
//
// Initialise state machine
//
this.Running = true;
this.ManagerStatus = StatusCodes.Running;
States state = States.sInit;
States lastState = States.sStart;
int lastEngineIndex = this.appData.farmSize - 1;
//
// State machine loop
//
try
{
bool success;
while (this.Running == true)
{
//
// Display message on each state change
//
if (state != lastState)
{
string logMessage = " [ " + STRLOG_MethodName + ": " + lastState.ToString() + " -> " + state.ToString() + " ]";
Logfile.Write(logMessage);
Trace.WriteLine(logMessage);
lastState = state;
}
switch (state)
{
case States.sInit:
//
// Update LabManager status
//
lock (this.statusLock)
{
this.slOnline = true;
this.slLabStatusMessage = StatusCodes.Ready.ToString();
}
//
// Revert any 'Running' experiments back to 'Waiting' so that they can be run again
//
ExperimentInfo[] allRunning = this.experimentQueue.RetrieveAllWithStatus(StatusCodes.Running);
for (int i = 0; i < allRunning.Length; i++)
{
int experimentId = allRunning[i].experimentId;
string sbName = allRunning[i].sbName;
success = this.experimentQueue.UpdateStatus(allRunning[i].experimentId, allRunning[i].sbName, StatusCodes.Waiting);
string logMessage = STRLOG_RevertingToWaiting + STRLOG_experimentId + experimentId.ToString() +
Logfile.STRLOG_Spacer + Logfile.STRLOG_Quote + STRLOG_sbName + sbName + Logfile.STRLOG_Quote +
Logfile.STRLOG_Spacer + STRLOG_success + success.ToString();
Logfile.Write(logMessage);
}
//
// Check if any experiments have not notified their ServiceBroker
//
this.submittedSinceLastNotifyCheck = true;
state = States.sIdle;
break;
case States.sIdle:
this.ManagerStatus = StatusCodes.Ready;
//
// Wait for an experiment to be submitted or timeout after QUEUE_CHECK_DELAY_SECS.
// In either case, check the experiment queue. Maybe an experiment submission
// signal got missed and it didn't get seen here. It has happened before.
//
lock (this.signalSubmitted)
{
if (Monitor.Wait(this.signalSubmitted, QUEUE_CHECK_DELAY_SECS * 1000) == true)
{
//
// Signal received, go check the queue
//
this.submittedSinceLastNotifyCheck = true;
state = States.sCheckExperimentQueue;
break;
}
}
//
// Timed out, go check some other things
//
if (this.submittedSinceLastNotifyCheck == true)
{
this.ManagerStatus = StatusCodes.Running;
state = States.sCheckNotified;
}
break;
case States.sCheckExperimentQueue:
//
// Wait a bit before checking the queue
//
Thread.Sleep(1000);
//
// Check the queue to see if there are any experiments waiting
//
if (this.experimentQueue.GetWaitCount() > 0)
{
state = States.sFindAvailableEngine;
break;
}
state = States.sIdle;
break;
case States.sFindAvailableEngine:
//
// Find an available experiment engine to run a waiting experiment
//
bool foundAvailableEngine = false;
for (int i = 0; i < this.appData.farmSize; i++)
{
//
// Determine which engine to look at
//
#if x
if (++lastEngineIndex == this.appData.farmSize)
{
lastEngineIndex = 0;
}
#else
lastEngineIndex = i;
#endif
//
// Determine if this experiment engine is currently running
//
LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[lastEngineIndex];
bool isRunning = labExperimentEngine.IsRunning;
string logMessage = " i: " + i.ToString() +
Logfile.STRLOG_Spacer + STRLOG_unitId + labExperimentEngine.UnitId.ToString() +
Logfile.STRLOG_Spacer + STRLOG_IsRunning + isRunning.ToString();
Logfile.Write(logMessage);
//
// Try starting the experiment engine, may already be running or offline
//
if (labExperimentEngine.Start() == true)
{
//
// An available engine has been found and started, check the queue again
//
foundAvailableEngine = true;
state = States.sCheckExperimentQueue;
break;
}
}
//
// Check if an available engine was found
//
if (foundAvailableEngine == false)
{
//
// Not found, have to wait for one
//
state = States.sWaitForAvailableEngine;
}
break;
case States.sWaitForAvailableEngine:
//
// Wait for an experiment engine to complete execution or timeout after ENGINE_CHECK_DELAY_SECS.
// In either case, check for an available engine. Maybe an experiment engine completion
// signal got missed and it didn't get seen here. It has happened before.
//
lock (this.signalCompleted)
{
if (Monitor.Wait(this.signalCompleted, ENGINE_CHECK_DELAY_SECS * 1000) == true)
{
//
// Signal received, find an available engine to run the next experiment
//
state = States.sFindAvailableEngine;
}
}
//
// No experiment engine has signalled its completion, check the queue again
//
state = States.sCheckExperimentQueue;
break;
case States.sCheckNotified:
//
// Check if any experiments have not notified their ServiceBroker
//
success = true;
ResultsIdInfo[] allNotNotified = this.experimentResults.RetrieveAllNotNotified();
for (int i = 0; i < allNotNotified.Length && success == true; i++)
{
int experimentId = allNotNotified[i].experimentId;
string sbName = allNotNotified[i].sbName;
try
{
//
// Attempt to notify the ServiceBroker for this experiment
//
LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.appData.allowedServiceBrokers);
if ((success = labServerToSbAPI.Notify(experimentId, sbName)) == true)
{
success = this.experimentResults.UpdateNotified(experimentId, sbName);
}
}
catch (Exception ex)
{
Logfile.WriteError(ex.Message);
success = false;
}
}
if (success == true)
{
//
// All notifies completed, don't check again until next experiment submission
//
this.submittedSinceLastNotifyCheck = false;
}
state = States.sCheckExperimentQueue;
break;
}
}
}
catch (Exception ex)
{
Logfile.WriteError(STRLOG_MethodName + ": " + ex.Message);
}
Trace.WriteLine(STRLOG_MethodName + ": Exiting");
Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName);
}
//-------------------------------------------------------------------------------------------------//
public LabStatus GetLabStatus()
{
LabStatus labStatus = null;
lock (this.managerLock)
{
bool isOnline;
string message;
//
// Get the lab status of the experiment manager
//
lock (this.statusLock)
{
isOnline = this.slOnline;
message = this.slLabStatusMessage;
}
//
// Check the lab status
//
if (isOnline == false)
{
labStatus = new LabStatus(isOnline, message);
}
else
{
isOnline = true;
message = String.Empty;
//
// Check lab status of each experiment engine
//
for (int i = 0; i < this.appData.farmSize; i++)
{
LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[i];
labStatus = labExperimentEngine.GetLabStatus();
//
// Keep a running tally
//
isOnline = isOnline && labStatus.online;
if (i > 0)
{
message += Logfile.STRLOG_Spacer;
}
message += i.ToString() + ":" + labStatus.labStatusMessage;
}
if (isOnline == true)
{
message = StatusCodes.Ready.ToString();
}
labStatus = new LabStatus(isOnline, message);
}
}
return(labStatus);
}