/// <summary>
        /// Function that gets fan speed requirements
        /// from all blades. It also updates the blade states.
        /// </summary>
        private static void GetAllBladePwmRequirements()
        {
            // Rate is required to timestep over each individual Blade call
            double rate     = (double)getBladePwmReqtTimePeriodInMilliseconds / (double)MaxSledCount;
            double timeDiff = 0;

            for (byte blade = 1; blade <= MaxSledCount; blade++)
            {
                // Handle shutdown state
                if (ChassisState.ShutDown)
                {
                    return;
                }

                // default PWM setting
                byte PWM = (byte)ConfigLoaded.MinPWM;

                // Query blade type from IPMI layer
                ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification;

                // wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement

                double sleepTime = rate - timeDiff;

                if (sleepTime > rate)
                {
                    sleepTime = rate;
                }
                if (sleepTime > 0)
                {
                    Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime));
                }

                Tracer.WriteInfo("GetBladeRequirement called at {0} for BladeId {1} (state: {2})", DateTime.Now, blade,
                                 ChassisState.GetStateName(blade));

                // Check for the condition where known state is hardpoweroff, but someone plugged a new blade in
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff)
                {
                    CheckPowerEnableState(blade);
                }

                // Log Start time
                DateTime startTime = DateTime.Now;

                #region Check fail State -> Initialize

                // If blade was in Fail state
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail)
                {
                    // If failed count is greater than a maximum value, we move it to Initialization state
                    if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount)
                    {
                        // Move to Initialization state so that this blade could be reinitialized
                        Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Initialization", blade,
                                         ChassisState.GetStateName(blade));
                        ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                    }
                    else
                    {
                        // Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there.
                        // do not allow retries on Get System Guid
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for blade {0}, GUID: {1}", blade, guid.Guid.ToString());

                            DeviceGuid cachedGuid = WcsBladeFacade.GetCachedGuid(blade);

                            if (guid.Guid == cachedGuid.Guid)
                            {
                                // Change state to Probation and assume the system was in fail due to timeout.
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade,
                                                 ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Probation);
                            }
                            else
                            {
                                // Change state to Initialization as the device has changed.
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade,
                                                 ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                            }
                        }
                        else
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID returns a bad completion status: {0}", guid.CompletionCode);
                        }
                    }

                    // Increase time spent in Fail state everytime we are in this state
                    ChassisState.FailCount[blade - 1]++;
                }

                #endregion

                #region Move Initialize -> Probation

                // Handles Initialization
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization)
                {
                    BladePowerStatePacket powerstate = ChassisState.BladePower[blade - 1].GetCachedBladePowerState();

                    if (powerstate.CompletionCode == 0)
                    {
                        if (powerstate.DecompressionTime == 0)
                        {
                            // Will result in Hard Power off or Probation
                            ReInitialize(blade);
                        }
                    }
                }

                #endregion


                // Normal operation - possible states are probation or healthy
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation ||
                    ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy)
                {
                    #region Jbod (no sensor reading)

                    if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod)
                    {
                        // Do not allow retries on system guid.
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for JBOD {0}, GUID: {1}",
                                             blade, guid.Guid.ToString());

                            // Change state to Healthy
                            if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                            {
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Healthy",
                                                 blade, ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            }
                        }
                        else
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID for JBOD {0} failed with status {1}",
                                             blade, guid.CompletionCode);
                            // Set it to failed state, where we will retry guids and reinitialize if needed
                            Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Fail",
                                             blade, ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                        }

                        // No need to check for sensor reading, just continue
                        continue;
                    }

                    #endregion

                    #region Server -> Get PWM move to Healthy or move to Fail

                    // Call temperature reading list command
                    SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System);

                    if (Temps.CompletionCode != (byte)CompletionCode.Success)
                    {
                        Tracer.WriteWarning("GetAllBladePwmRequirements: BladeId: {0} - GetSensorReading for temperature failed with code {1:X}",
                                            blade, Temps.CompletionCode);

                        // Move to Fail state if no readings were obtained
                        Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Fail", blade,
                                         ChassisState.GetStateName(blade));

                        ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                    }
                    else
                    {
                        Tracer.WriteInfo("GetAllBladePwmRequirements: #### BladeId = " + blade + " Sensor id= " + ConfigLoaded.InputSensor +
                                         " Sensor reading = " + Temps.Reading + " Raw = " + Temps.RawReading +
                                         ", LowerNonCritical= " + ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold);

                        // Handle state logic if needed
                        // Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded
                        if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                        {
                            // Change state to healthy
                            Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Healthy",
                                             blade, ChassisState.GetStateName(blade));

                            ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            ChassisState.FailCount[blade - 1] = 0; // reset the fail count

                            // When a blade transitions to 'Healthy' state, enable/disable default blade operations
                            EnableDisableDefaultBladeOperations(blade);
                        }

                        if (ConfigLoaded.InputSensor != 1) // Non-PWM sensor.
                        {
                            PWM = GetPwmFromTemperature(Temps.Reading,
                                                        ConfigLoaded.SensorLowThreshold,
                                                        ConfigLoaded.SensorHighThreshold);
                        }
                        else
                        {
                            // PWM should never be higher or lower than the threshold.
                            if (Temps.Reading < ConfigLoaded.MinPWM || Temps.Reading > ConfigLoaded.MaxPWM)
                            {
                                Tracer.WriteWarning("PWM value " + Temps.Reading + " on blade " + blade +
                                                    " is out of range (lowThreshold: " + ConfigLoaded.MinPWM +
                                                    " - highThreshold: " + ConfigLoaded.MaxPWM);

                                PWM = (byte)ConfigLoaded.MinPWM;
                            }
                            else
                            {
                                PWM = (byte)Temps.Reading;
                            }
                        }

                        Tracer.WriteInfo("PWM value on blade {0} for Sensor {1} = {2}", blade, InputSensor, PWM);
                    }

                    #endregion
                }

                // write value into requirements table
                BladeRequirementTable[blade - 1] = PWM;

                // Log end time and capture time of execution for sensor get command
                DateTime endTime = DateTime.Now;
                timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds
            }
        }
Esempio n. 2
0
        /// <summary>
        /// Function that gets all the fan speed requirements
        /// from the Blade.  It also updates the balde state
        /// </summary>
        private void GetAllBladePwmRequirements()
        {
            // Rate is required to timestep over each individual Blade call
            double rate     = (double)GetTimePeriod / (double)MaxSledCount;
            double timeDiff = 0;

            for (byte blade = 1; blade <= MaxSledCount; blade++)
            {
                // Handle shutdown state
                if (ChassisState.ShutDown)
                {
                    return;
                }

                // default PWM setting
                byte PWM = (byte)ConfigLoaded.MinPWM;

                // Query blade type from IPMI layer
                ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification;

                // wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement

                double sleepTime = rate - timeDiff;

                if (sleepTime > rate)
                {
                    sleepTime = rate;
                }
                if (sleepTime > 0)
                {
                    Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime));
                }
                if (CommunicationDevice.IsSafeMode())
                {
                    // Do not perform any sensor reading - continue in the for loop
                    Tracer.WriteInfo("Monitoring thread: Safe Mode, Skipping sensor read");
                    continue;
                }
                Tracer.WriteInfo("GetBladeRequirement called at {0} for sledId {1} (state: {2})", DateTime.Now, blade,
                                 ChassisState.GetStateName(blade));

                // Check for the condition where known state is hardpoweroff, but someone plugged a new blade in
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff)
                {
                    ChassisState.PowerFailCount[blade - 1]++;
                    // TODO: identify if this period is sufficient to do this check
                    if (ChassisState.PowerFailCount[blade - 1] > (ConfigLoaded.MaxRetries * ConfigLoaded.Population))
                    {
                        CheckPowerEnableState(blade);
                        ChassisState.PowerFailCount[blade - 1] = 0;
                    }
                }
                // Log Start time
                DateTime startTime = DateTime.Now;

                // If blade was in Fail state
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail)
                {
                    // If failed count is greater than a maximum value, we move it to Initialization state

                    if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount)
                    {
                        // Move to Initialization state so that this sled could be reinitialized
                        Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Initialization", blade,
                                         ChassisState.GetStateName(blade));
                        ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                    }
                    else
                    {
                        // Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GUID present for sled {0}, GUID: {1}", blade, guid.Guid.ToString());

                            // Change state to Probation
                            Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", blade,
                                             ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Probation);
                        }
                        else
                        {
                            Tracer.WriteInfo("Get System GUID returns a bad completion status: {0}", guid.CompletionCode);
                        }
                    }
                    // Increase time spent in Fail state everytime we are in this state
                    ChassisState.FailCount[blade - 1]++;
                }

                // Handles Initialization
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization)
                {
                    this.ReInitialize(blade);
                }

                // Normal operation - possible states are probation or healthy
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation ||
                    ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy)
                {
                    if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod)
                    {
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GUID present for jbod {0}, GUID: {1}", blade, guid.Guid.ToString());

                            // Change state to Probation
                            Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Healthy", blade, ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                        }
                        else
                        {
                            Tracer.WriteInfo("Get System GUID for jbod {0} failed with status {1}", blade, guid.CompletionCode);
                            // Set it to failed state, where we will retry guids and reinitialize if needed
                            Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Fail", blade, ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                        }

                        // No need to check for sensor reading, just continue
                        continue;
                    }
                    // Call temperature reading list command
                    SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System);

                    if (Temps.CompletionCode != (byte)CompletionCode.Success)
                    {
                        Tracer.WriteWarning("SledId: {0} - getTempSensorReading failed with code {1:X}", blade, Temps.CompletionCode);

                        // Move to Fail state if no readings were obtained
                        Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Fail", blade,
                                         ChassisState.GetStateName(blade));

                        ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                    }
                    else
                    {
                        Tracer.WriteInfo("#### Sledid= " + blade + " Sensor id= " + ConfigLoaded.InputSensor + " Sensor reading= " +
                                         Temps.Reading + " Raw= " + Temps.RawReading + ", LowerNonCritical= " +
                                         ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold);

                        // Handle state logic if needed
                        // Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded
                        if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                        {
                            // Change state to healthy
                            Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Healthy", blade,
                                             ChassisState.GetStateName(blade));

                            ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            ChassisState.FailCount[blade - 1] = 0; // reset the fail count
                        }

                        PWM = GetPwmFromTemperature(Temps.Reading,
                                                    ConfigLoaded.SensorLowThreshold,
                                                    ConfigLoaded.SensorHighThreshold);

                        Tracer.WriteInfo("PWM value for Sensor {0} = {1}", InputSensor, PWM);
                    }
                }

                // write value into requirements table
                BladeRequirementTable[blade - 1] = PWM;

                // Log end time and capture time of execution for sensor get command
                DateTime endTime = DateTime.Now;
                timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds
            }
        }