//-------------------------------------------------------------------------------------------------//

        private bool NotifyServiceBroker(ExperimentInfo experimentInfo)
        {
            const string STRLOG_MethodName = "NotifyServiceBroker";

            Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);

            bool success = false;

            try
            {
                //
                // Notify the ServiceBroker so that the results can be retrieved
                //
                LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.allowedServiceBrokers);
                if ((success = labServerToSbAPI.Notify(experimentInfo.experimentId, experimentInfo.sbName)) == true)
                {
                    success = this.experimentResults.UpdateNotified(experimentInfo.experimentId, experimentInfo.sbName);
                }
            }
            catch (Exception ex)
            {
                Logfile.WriteError(ex.Message);
            }

            string logMessage = STRLOG_success + success.ToString();

            Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName, logMessage);

            return(success);
        }
        //-------------------------------------------------------------------------------------------------//
        private bool NotifyServiceBroker(ExperimentInfo experimentInfo)
        {
            const string STRLOG_MethodName = "NotifyServiceBroker";

            Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);

            bool success = false;

            try
            {
                //
                // Notify the ServiceBroker so that the results can be retrieved
                //
                LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.allowedServiceBrokers);
                if ((success = labServerToSbAPI.Notify(experimentInfo.experimentId, experimentInfo.sbName)) == true)
                {
                    success = this.experimentResults.UpdateNotified(experimentInfo.experimentId, experimentInfo.sbName);
                }
            }
            catch (Exception ex)
            {
                Logfile.WriteError(ex.Message);
            }

            string logMessage = STRLOG_success + success.ToString();

            Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName, logMessage);

            return success;
        }
        //-------------------------------------------------------------------------------------------------//
        private void LabExperimentManagerThread()
        {
            const string STRLOG_MethodName = "LabExperimentManagerThread";

            Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);

            //
            // Initialise state machine
            //
            this.Running = true;
            this.ManagerStatus = StatusCodes.Running;
            States state = States.sInit;
            States lastState = States.sStart;
            int lastEngineIndex = this.appData.farmSize - 1;

            //
            // State machine loop
            //
            try
            {
                bool success;

                while (this.Running == true)
                {
                    //
                    // Display message on each state change
                    //
                    if (state != lastState)
                    {
                        string logMessage = " [ " + STRLOG_MethodName + ": " + lastState.ToString() + " -> " + state.ToString() + " ]";
                        Logfile.Write(logMessage);
                        Trace.WriteLine(logMessage);

                        lastState = state;
                    }

                    switch (state)
                    {
                        case States.sInit:

                            //
                            // Update LabManager status
                            //
                            lock (this.statusLock)
                            {
                                this.slOnline = true;
                                this.slLabStatusMessage = StatusCodes.Ready.ToString();
                            }

                            //
                            // Revert any 'Running' experiments back to 'Waiting' so that they can be run again
                            //
                            ExperimentInfo[] allRunning = this.experimentQueue.RetrieveAllWithStatus(StatusCodes.Running);
                            for (int i = 0; i < allRunning.Length; i++)
                            {
                                int experimentId = allRunning[i].experimentId;
                                string sbName = allRunning[i].sbName;

                                success = this.experimentQueue.UpdateStatus(allRunning[i].experimentId, allRunning[i].sbName, StatusCodes.Waiting);

                                string logMessage = STRLOG_RevertingToWaiting + STRLOG_experimentId + experimentId.ToString() +
                                    Logfile.STRLOG_Spacer + Logfile.STRLOG_Quote + STRLOG_sbName + sbName + Logfile.STRLOG_Quote +
                                    Logfile.STRLOG_Spacer + STRLOG_success + success.ToString();

                                Logfile.Write(logMessage);
                            }

                            //
                            // Check if any experiments have not notified their ServiceBroker
                            //
                            this.submittedSinceLastNotifyCheck = true;
                            state = States.sIdle;
                            break;

                        case States.sIdle:

                            this.ManagerStatus = StatusCodes.Ready;

                            //
                            // Wait for an experiment to be submitted or timeout after QUEUE_CHECK_DELAY_SECS.
                            // In either case, check the experiment queue. Maybe an experiment submission
                            // signal got missed and it didn't get seen here. It has happened before.
                            //
                            lock (this.signalSubmitted)
                            {
                                if (Monitor.Wait(this.signalSubmitted, QUEUE_CHECK_DELAY_SECS * 1000) == true)
                                {
                                    //
                                    // Signal received, go check the queue
                                    //
                                    this.submittedSinceLastNotifyCheck = true;
                                    state = States.sCheckExperimentQueue;
                                    break;
                                }
                            }

                            //
                            // Timed out, go check some other things
                            //
                            if (this.submittedSinceLastNotifyCheck == true)
                            {
                                this.ManagerStatus = StatusCodes.Running;
                                state = States.sCheckNotified;
                            }
                            break;

                        case States.sCheckExperimentQueue:

                            //
                            // Wait a bit before checking the queue
                            //
                            Thread.Sleep(1000);

                            //
                            // Check the queue to see if there are any experiments waiting
                            //
                            if (this.experimentQueue.GetWaitCount() > 0)
                            {
                                state = States.sFindAvailableEngine;
                                break;
                            }

                            state = States.sIdle;
                            break;

                        case States.sFindAvailableEngine:

                            //
                            // Find an available experiment engine to run a waiting experiment
                            //
                            bool foundAvailableEngine = false;
                            for (int i = 0; i < this.appData.farmSize; i++)
                            {
                                //
                                // Determine which engine to look at
                                //
            #if x
                                if (++lastEngineIndex == this.appData.farmSize)
                                {
                                    lastEngineIndex = 0;
                                }
            #else
                                lastEngineIndex = i;
            #endif
                                //
                                // Determine if this experiment engine is currently running
                                //
                                LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[lastEngineIndex];
                                bool isRunning = labExperimentEngine.IsRunning;

                                string logMessage = " i: " + i.ToString() +
                                    Logfile.STRLOG_Spacer + STRLOG_unitId + labExperimentEngine.UnitId.ToString() +
                                    Logfile.STRLOG_Spacer + STRLOG_IsRunning + isRunning.ToString();
                                Logfile.Write(logMessage);

                                //
                                // Try starting the experiment engine, may already be running or offline
                                //
                                if (labExperimentEngine.Start() == true)
                                {
                                    //
                                    // An available engine has been found and started, check the queue again
                                    //
                                    foundAvailableEngine = true;
                                    state = States.sCheckExperimentQueue;
                                    break;
                                }
                            }

                            //
                            // Check if an available engine was found
                            //
                            if (foundAvailableEngine == false)
                            {
                                //
                                // Not found, have to wait for one
                                //
                                state = States.sWaitForAvailableEngine;
                            }
                            break;

                        case States.sWaitForAvailableEngine:

                            //
                            // Wait for an experiment engine to complete execution or timeout after ENGINE_CHECK_DELAY_SECS.
                            // In either case, check for an available engine. Maybe an experiment engine completion
                            // signal got missed and it didn't get seen here. It has happened before.
                            //
                            lock (this.signalCompleted)
                            {
                                if (Monitor.Wait(this.signalCompleted, ENGINE_CHECK_DELAY_SECS * 1000) == true)
                                {
                                    //
                                    // Signal received, find an available engine to run the next experiment
                                    //
                                    state = States.sFindAvailableEngine;
                                }
                            }

                            //
                            // No experiment engine has signalled its completion, check the queue again
                            //
                            state = States.sCheckExperimentQueue;
                            break;

                        case States.sCheckNotified:

                            //
                            // Check if any experiments have not notified their ServiceBroker
                            //
                            success = true;
                            ResultsIdInfo[] allNotNotified = this.experimentResults.RetrieveAllNotNotified();

                            for (int i = 0; i < allNotNotified.Length && success == true; i++)
                            {
                                int experimentId = allNotNotified[i].experimentId;
                                string sbName = allNotNotified[i].sbName;

                                try
                                {
                                    //
                                    // Attempt to notify the ServiceBroker for this experiment
                                    //
                                    LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.appData.allowedServiceBrokers);
                                    if ((success = labServerToSbAPI.Notify(experimentId, sbName)) == true)
                                    {
                                        success = this.experimentResults.UpdateNotified(experimentId, sbName);
                                    }
                                }
                                catch (Exception ex)
                                {
                                    Logfile.WriteError(ex.Message);
                                    success = false;
                                }
                            }

                            if (success == true)
                            {
                                //
                                // All notifies completed, don't check again until next experiment submission
                                //
                                this.submittedSinceLastNotifyCheck = false;
                            }
                            state = States.sCheckExperimentQueue;
                            break;
                    }
                }
            }
            catch (Exception ex)
            {
                Logfile.WriteError(STRLOG_MethodName + ": " + ex.Message);
            }

            Trace.WriteLine(STRLOG_MethodName + ": Exiting");

            Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName);
        }
Beispiel #4
0
        //-------------------------------------------------------------------------------------------------//

        private void LabExperimentManagerThread()
        {
            const string STRLOG_MethodName = "LabExperimentManagerThread";

            Logfile.WriteCalled(STRLOG_ClassName, STRLOG_MethodName);

            //
            // Initialise state machine
            //
            this.Running       = true;
            this.ManagerStatus = StatusCodes.Running;
            States state           = States.sInit;
            States lastState       = States.sStart;
            int    lastEngineIndex = this.appData.farmSize - 1;

            //
            // State machine loop
            //
            try
            {
                bool success;

                while (this.Running == true)
                {
                    //
                    // Display message on each state change
                    //
                    if (state != lastState)
                    {
                        string logMessage = " [ " + STRLOG_MethodName + ": " + lastState.ToString() + " -> " + state.ToString() + " ]";
                        Logfile.Write(logMessage);
                        Trace.WriteLine(logMessage);

                        lastState = state;
                    }

                    switch (state)
                    {
                    case States.sInit:

                        //
                        // Update LabManager status
                        //
                        lock (this.statusLock)
                        {
                            this.slOnline           = true;
                            this.slLabStatusMessage = StatusCodes.Ready.ToString();
                        }

                        //
                        // Revert any 'Running' experiments back to 'Waiting' so that they can be run again
                        //
                        ExperimentInfo[] allRunning = this.experimentQueue.RetrieveAllWithStatus(StatusCodes.Running);
                        for (int i = 0; i < allRunning.Length; i++)
                        {
                            int    experimentId = allRunning[i].experimentId;
                            string sbName       = allRunning[i].sbName;

                            success = this.experimentQueue.UpdateStatus(allRunning[i].experimentId, allRunning[i].sbName, StatusCodes.Waiting);

                            string logMessage = STRLOG_RevertingToWaiting + STRLOG_experimentId + experimentId.ToString() +
                                                Logfile.STRLOG_Spacer + Logfile.STRLOG_Quote + STRLOG_sbName + sbName + Logfile.STRLOG_Quote +
                                                Logfile.STRLOG_Spacer + STRLOG_success + success.ToString();

                            Logfile.Write(logMessage);
                        }

                        //
                        // Check if any experiments have not notified their ServiceBroker
                        //
                        this.submittedSinceLastNotifyCheck = true;
                        state = States.sIdle;
                        break;

                    case States.sIdle:

                        this.ManagerStatus = StatusCodes.Ready;

                        //
                        // Wait for an experiment to be submitted or timeout after QUEUE_CHECK_DELAY_SECS.
                        // In either case, check the experiment queue. Maybe an experiment submission
                        // signal got missed and it didn't get seen here. It has happened before.
                        //
                        lock (this.signalSubmitted)
                        {
                            if (Monitor.Wait(this.signalSubmitted, QUEUE_CHECK_DELAY_SECS * 1000) == true)
                            {
                                //
                                // Signal received, go check the queue
                                //
                                this.submittedSinceLastNotifyCheck = true;
                                state = States.sCheckExperimentQueue;
                                break;
                            }
                        }

                        //
                        // Timed out, go check some other things
                        //
                        if (this.submittedSinceLastNotifyCheck == true)
                        {
                            this.ManagerStatus = StatusCodes.Running;
                            state = States.sCheckNotified;
                        }
                        break;

                    case States.sCheckExperimentQueue:

                        //
                        // Wait a bit before checking the queue
                        //
                        Thread.Sleep(1000);

                        //
                        // Check the queue to see if there are any experiments waiting
                        //
                        if (this.experimentQueue.GetWaitCount() > 0)
                        {
                            state = States.sFindAvailableEngine;
                            break;
                        }

                        state = States.sIdle;
                        break;

                    case States.sFindAvailableEngine:

                        //
                        // Find an available experiment engine to run a waiting experiment
                        //
                        bool foundAvailableEngine = false;
                        for (int i = 0; i < this.appData.farmSize; i++)
                        {
                            //
                            // Determine which engine to look at
                            //
#if x
                            if (++lastEngineIndex == this.appData.farmSize)
                            {
                                lastEngineIndex = 0;
                            }
#else
                            lastEngineIndex = i;
#endif
                            //
                            // Determine if this experiment engine is currently running
                            //
                            LabExperimentEngine labExperimentEngine = this.appData.labExperimentEngines[lastEngineIndex];
                            bool isRunning = labExperimentEngine.IsRunning;

                            string logMessage = " i: " + i.ToString() +
                                                Logfile.STRLOG_Spacer + STRLOG_unitId + labExperimentEngine.UnitId.ToString() +
                                                Logfile.STRLOG_Spacer + STRLOG_IsRunning + isRunning.ToString();
                            Logfile.Write(logMessage);

                            //
                            // Try starting the experiment engine, may already be running or offline
                            //
                            if (labExperimentEngine.Start() == true)
                            {
                                //
                                // An available engine has been found and started, check the queue again
                                //
                                foundAvailableEngine = true;
                                state = States.sCheckExperimentQueue;
                                break;
                            }
                        }

                        //
                        // Check if an available engine was found
                        //
                        if (foundAvailableEngine == false)
                        {
                            //
                            // Not found, have to wait for one
                            //
                            state = States.sWaitForAvailableEngine;
                        }
                        break;

                    case States.sWaitForAvailableEngine:

                        //
                        // Wait for an experiment engine to complete execution or timeout after ENGINE_CHECK_DELAY_SECS.
                        // In either case, check for an available engine. Maybe an experiment engine completion
                        // signal got missed and it didn't get seen here. It has happened before.
                        //
                        lock (this.signalCompleted)
                        {
                            if (Monitor.Wait(this.signalCompleted, ENGINE_CHECK_DELAY_SECS * 1000) == true)
                            {
                                //
                                // Signal received, find an available engine to run the next experiment
                                //
                                state = States.sFindAvailableEngine;
                            }
                        }

                        //
                        // No experiment engine has signalled its completion, check the queue again
                        //
                        state = States.sCheckExperimentQueue;
                        break;

                    case States.sCheckNotified:

                        //
                        // Check if any experiments have not notified their ServiceBroker
                        //
                        success = true;
                        ResultsIdInfo[] allNotNotified = this.experimentResults.RetrieveAllNotNotified();


                        for (int i = 0; i < allNotNotified.Length && success == true; i++)
                        {
                            int    experimentId = allNotNotified[i].experimentId;
                            string sbName       = allNotNotified[i].sbName;

                            try
                            {
                                //
                                // Attempt to notify the ServiceBroker for this experiment
                                //
                                LabServerToSbAPI labServerToSbAPI = new LabServerToSbAPI(this.appData.allowedServiceBrokers);
                                if ((success = labServerToSbAPI.Notify(experimentId, sbName)) == true)
                                {
                                    success = this.experimentResults.UpdateNotified(experimentId, sbName);
                                }
                            }
                            catch (Exception ex)
                            {
                                Logfile.WriteError(ex.Message);
                                success = false;
                            }
                        }

                        if (success == true)
                        {
                            //
                            // All notifies completed, don't check again until next experiment submission
                            //
                            this.submittedSinceLastNotifyCheck = false;
                        }
                        state = States.sCheckExperimentQueue;
                        break;
                    }
                }
            }
            catch (Exception ex)
            {
                Logfile.WriteError(STRLOG_MethodName + ": " + ex.Message);
            }

            Trace.WriteLine(STRLOG_MethodName + ": Exiting");

            Logfile.WriteCompleted(STRLOG_ClassName, STRLOG_MethodName);
        }