private BladePowerStatePacket GetBladePowerState(byte deviceId)
        {
            // Initialize return packet
            BladePowerStatePacket returnPacket = new BladePowerStatePacket();

            returnPacket.CompletionCode  = CompletionCode.UnspecifiedError;
            returnPacket.BladePowerState = 3;

            try
            {
                BladePowerStateResponse stateResponse = (BladePowerStateResponse)this.SendReceive(this.DeviceType, deviceId, new BladePowerStateRequest(), typeof(BladePowerStateResponse));
                if (stateResponse.CompletionCode != 0)
                {
                    returnPacket.CompletionCode = (CompletionCode)stateResponse.CompletionCode;
                }
                else
                {
                    returnPacket.CompletionCode    = CompletionCode.Success;
                    returnPacket.BladePowerState   = stateResponse.BladePowerState;
                    returnPacket.DecompressionTime = stateResponse.DecompressionTime;
                }
            }
            catch (System.Exception ex)
            {
                returnPacket.CompletionCode  = CompletionCode.UnspecifiedError;
                returnPacket.BladePowerState = 2;
                Tracer.WriteError(this.DeviceId, this.DeviceType, ex);
            }
            return(returnPacket);
        }
        private BladePowerStatePacket SetBladePowerState(byte deviceId, byte state)
        {
            // Initialize return packet
            BladePowerStatePacket returnPacket = new BladePowerStatePacket();

            returnPacket.CompletionCode  = CompletionCode.UnspecifiedError;
            returnPacket.BladePowerState = 3;

            Tracer.WriteInfo("SetSledPowerState Switch id: " + deviceId);

            if (state == (byte)Contracts.PowerState.ON)
            {
                try
                {
                    BladePowerStateOnResponse response = (BladePowerStateOnResponse)this.SendReceive(this.DeviceType, deviceId, new BladePowerStateOnRequest(state), typeof(BladePowerStateOnResponse));
                    if (response.CompletionCode != 0)
                    {
                        returnPacket.CompletionCode = (CompletionCode)response.CompletionCode;
                    }
                    else
                    {
                        returnPacket.CompletionCode    = CompletionCode.Success;
                        returnPacket.BladePowerState   = 1; //NOTE: response is not actually returned.
                        returnPacket.DecompressionTime = response.DecompressionTime;
                    }
                }
                catch (System.Exception ex)
                {
                    returnPacket.CompletionCode  = CompletionCode.UnspecifiedError;
                    returnPacket.BladePowerState = 3;
                    Tracer.WriteError(this.DeviceId, DeviceType.Power, ex);
                }
            }
            else
            {
                try
                {
                    BladePowerStateOffResponse response = (BladePowerStateOffResponse)this.SendReceive(this.DeviceType, deviceId, new BladePowerStateOffRequest(state), typeof(BladePowerStateOffResponse));
                    if (response.CompletionCode != 0)
                    {
                        returnPacket.CompletionCode = (CompletionCode)response.CompletionCode;
                    }
                    else
                    {
                        returnPacket.CompletionCode    = CompletionCode.Success;
                        returnPacket.BladePowerState   = 0; // blade is off
                        returnPacket.DecompressionTime = 0; // blade is off.
                    }
                }
                catch (System.Exception ex)
                {
                    returnPacket.CompletionCode  = CompletionCode.UnspecifiedError;
                    returnPacket.BladePowerState = 3;
                    Tracer.WriteError(this.DeviceId, DeviceType.Power, ex);
                }
            }

            return(returnPacket);
        }
Example #3
0
        /// <summary>
        /// Reinitialize the sled and set chassis state
        /// </summary>
        private void ReInitialize(byte sledId)
        {
            // Serialize initialize and power behavior per sled
            lock (ChassisState._lock[sledId - 1])
            {
                ChassisState.FailCount[sledId - 1] = 0;                // reset fail count since we are going to reinitialize the blade

                bool status = WcsBladeFacade.InitializeClient(sledId); // TODO: no completion code, only byte status returned

                if (status != true)
                {
                    // Initialization failed - move to fail state before retrying again
                    Tracer.WriteInfo("Reinitialization failed with code: {0} for Sled: {1}", status, sledId);
                    Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Fail", sledId,
                                     ChassisState.GetStateName(sledId));

                    ChassisState.SetBladeState((byte)sledId, (byte)BladeState.Fail);

                    // check power status to see if the blade was manually switched off or removed
                    BladePowerStatePacket response = ChassisState.BladePower[sledId - 1].GetBladePowerState();

                    // If the blade was turned off, set correct status / TODO: do we need this here?
                    if (response.BladePowerState == (byte)Contracts.PowerState.OFF)
                    {
                        Tracer.WriteInfo("SledId {0} is in hard power off state", sledId);
                        Tracer.WriteInfo("State Transition for Sled {0}: {1} -> HardPowerOff", sledId,
                                         ChassisState.GetStateName(sledId));

                        ChassisState.SetBladeState(sledId, (byte)BladeState.HardPowerOff);
                    }
                }
                else
                {
                    // State change: I -> P
                    Tracer.WriteInfo("Reinitialization of Sled: {0} succeeded with status {1}", sledId, status);

                    Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", sledId,
                                     ChassisState.GetStateName(sledId));
                    ChassisState.SetBladeState(sledId, (byte)BladeState.Probation);

                    // Initialize Blade Type (Type might have changed when Blades were reinserted)
                    if (WcsBladeFacade.clients.ContainsKey(sledId))
                    {
                        ChassisState.BladeTypeCache[sledId - 1] = (byte)WcsBladeFacade.clients[sledId].BladeClassification;
                    }
                    else
                    {
                        ChassisState.BladeTypeCache[sledId - 1] = (byte)BladeType.Unknown;
                    }
                }
            }
        }
Example #4
0
        /// <summary>
        /// Internal operation to call both hard power on (soft power on is not exposed to the user)
        /// </summary>
        /// <param name="bladeId">Blade ID</param>
        /// <returns>True/false for success/failure</returns>
        internal static bool PowerOn(int bladeId)
        {
            Tracer.WriteInfo("Received poweron({0})", bladeId);
            bool powerOnStatus = false;

            BladePowerStatePacket bladePowerSwitchStatePacket = new BladePowerStatePacket();
            CompletionCode        status;

            // Hard Power enable
            // Serialize setting of state and actual code logic
            lock (ChassisState.locker[bladeId - 1])
            {
                BladePowerStatePacket currState = ChassisState.BladePower[bladeId - 1].GetBladePowerState();

                if (currState.CompletionCode != CompletionCode.Success ||
                    (currState.BladePowerState == (byte)Contracts.PowerState.OFF))
                {
                    // No return here, because we still want to return a BMC state on the fall through,
                    // if Blade enable read fails for whatever reason
                    Tracer.WriteWarning("PowerOn: Blade {0} Power Enable state read failed (Completion Code: {1:X})", bladeId, currState.CompletionCode);

                    bladePowerSwitchStatePacket = ChassisState.BladePower[bladeId - 1].SetBladePowerState((byte)PowerState.ON);
                    status = bladePowerSwitchStatePacket.CompletionCode;
                    Tracer.WriteInfo("Hard poweron status " + status);

                    if (status == CompletionCode.Success)
                    {
                        // Hard power on status is true, so Blade should be set to Initialization state on success
                        Tracer.WriteInfo("PowerOn: State Transition for blade {0}: {1} -> Initialization", bladeId,
                                         ChassisState.GetStateName((byte)bladeId));

                        ChassisState.SetBladeState((byte)bladeId, (byte)BladeState.Initialization);
                        powerOnStatus = true;
                    }
                    else
                    {
                        Tracer.WriteWarning("PowerOn: Hard Power On failed for BladeId {0} with code {1:X}", bladeId, status);
                    }
                }
                else
                {
                    powerOnStatus = true; // the blade was already powered on, so we dont power it on again
                }
            }
            return(powerOnStatus);
        }
        /// <summary>
        /// Checks the power enable state of the blade and changes state accordingly
        /// </summary>
        /// <param name="deviceId"></param>
        private static void CheckPowerEnableState(byte deviceId)
        {
            // Serialize power behavior
            lock (ChassisState.locker[deviceId - 1])
            {
                BladePowerStatePacket response = ChassisState.BladePower[deviceId - 1].GetCachedBladePowerState();

                if (response.CompletionCode != CompletionCode.Success)
                {
                    Tracer.WriteInfo("CheckPowerEnableState: Blade {0} Power Enable state read failed (Completion Code: {1:X})",
                                     deviceId, response.CompletionCode);
                }
                else
                {
                    if (response.BladePowerState == (byte)Contracts.PowerState.ON)
                    {
                        if (ChassisState.GetBladeState((byte)deviceId) == (byte)BladeState.HardPowerOff)
                        {
                            // Blade is powered on, move to initialization state
                            Tracer.WriteInfo("CheckPowerEnableState: State Transition for blade {0}: {1} -> Initialization",
                                             deviceId, ChassisState.GetStateName(deviceId));
                            ChassisState.SetBladeState((byte)deviceId, (byte)BladeState.Initialization);
                        }
                    }
                    else if (response.BladePowerState == (byte)Contracts.PowerState.OFF)
                    {
                        if (ChassisState.GetBladeState((byte)deviceId) != (byte)BladeState.HardPowerOff)
                        {
                            // Blade is powered off, move to PowerOff state
                            Tracer.WriteInfo("CheckPowerEnableState: State Transition for blade {0}: {1} -> HardPowerOff",
                                             deviceId, ChassisState.GetStateName(deviceId));

                            ChassisState.SetBladeState((byte)deviceId, (byte)BladeState.HardPowerOff);
                        }
                    }
                    else
                    {
                        Tracer.WriteInfo("CheckPowerEnableState: Getting out of else block");
                        // TODO: do we need to do anything for state that is NA
                    }
                }
            }
        }
        private static void EnableDisableDefaultBladeOperations(int bladeId)
        {
            // TODO: Check blade type etc and Kill any serial session
            // TODO: Add trace log messages

            // Check to see if the blade is hard powered off
            BladePowerStatePacket response = ChassisState.BladePower[bladeId - 1].GetBladePowerState();

            if (response.CompletionCode != CompletionCode.Success)
            {
                // Log error here, and proceed to check blade state since we still want to check BMC soft power status
                // even if blade enable read failed for whatever reason
                Tracer.WriteError("EnableDisableDefaultBladeOperations: Blade {0} Power Enable state read failed (Completion Code: {1:X})",
                                  bladeId, response.CompletionCode);
            }
            else if (response.BladePowerState == (byte)Contracts.PowerState.OFF)
            {
                // If blade is hard powered off, no further processing is necessary
                return;
            }

            // If the blade is a Jbod, return since the operations done in this method do not apply for Jbods
            if (ChassisState.GetBladeType((byte)bladeId) == (byte)BladeType.Jbod)
            {
                Tracer.WriteInfo("EnableDisableDefaultBladeOperations (Blade#{0}): Ignoring since it is a Jbod", bladeId);
                return;
            }

            DatasafeOperationSupport.ProcessDatasafeAction(bladeId, ConfigLoaded.DatasafeOperationsEnabled ? DatasafeActions.EnableDatasafe :
                                                           DatasafeActions.DisableDatasafe);

            if (ConfigLoaded.PsuAlertMonitorEnabled)
            {
                WcsBladeFacade.ActivatePsuAlert((byte)bladeId, true, BmcPsuAlertAction.ProcHotAndDpc, true);
            }
            else
            {
                WcsBladeFacade.ActivatePsuAlert((byte)bladeId, false, BmcPsuAlertAction.NoAction, true);
            }
        }
Example #7
0
        /// <summary>
        /// Internal method to Power off blade
        /// </summary>
        /// <param name="bladeId">Blade ID(1-48)</param>
        /// <returns>true/false if operation was success/failure</returns>
        internal static bool PowerOff(int bladeId)
        {
            Tracer.WriteInfo("Received poweroff({0})", bladeId);
            bool powerOffStatus = false;

            BladePowerStatePacket bladePowerSwitchStatePacket = new BladePowerStatePacket();

            // Serialize power off and power on, on the same lock variable per blade, so we prevent inconsistent power state behavior
            lock (ChassisState.locker[bladeId - 1])
            {
                bladePowerSwitchStatePacket = ChassisState.BladePower[bladeId - 1].SetBladePowerState((byte)PowerState.OFF);
                CompletionCode status = bladePowerSwitchStatePacket.CompletionCode;

                // Sleep for specified amount of time after blade hard power off to prevent hardware inconsistent state
                // - hot-swap controller not completely draining its capacitance leading to inconsistent power state issues
                Thread.Sleep(ConfigLoaded.WaitTimeAfterBladeHardPowerOffInMsecs);

                Tracer.WriteInfo("PowerOff: Return: {0}", status);

                if (status != CompletionCode.Success)
                {
                    Tracer.WriteError("PowerOff: Blade Hard Power Off Failed with Completion code {0:X}", status);
                    powerOffStatus = false;
                }
                else
                {
                    powerOffStatus = true;
                    // set state to Hard Power Off
                    Tracer.WriteInfo("PowerOff: State Transition for blade {0}: {1} -> HardPowerOff", bladeId,
                                     ChassisState.GetStateName((byte)bladeId));

                    ChassisState.SetBladeState((byte)bladeId, (byte)BladeState.HardPowerOff);
                    ChassisState.PowerFailCount[bladeId - 1] = 0;
                    // Clear blade type and cache
                    ChassisState.BladeTypeCache[bladeId - 1] = (byte)BladeType.Unknown;
                    WcsBladeFacade.ClearBladeClassification((byte)bladeId);
                }
            }
            return(powerOffStatus);
        }
        /// <summary>
        /// Internal operation to call both hard power on (soft power on is not exposed to the user)
        /// </summary>
        /// <param name="bladeId">Blade ID</param>
        /// <returns>True/false for success/failure</returns>
        internal static bool PowerOn(int bladeId)
        {
            Tracer.WriteInfo("Received poweron({0})", bladeId);
            bool powerOnStatus = false;

            BladePowerStatePacket bladePowerSwitchStatePacket = new BladePowerStatePacket();
            CompletionCode status;

            // Hard Power enable
            // Serialize setting of state and actual code logic
            lock (ChassisState.locker[bladeId - 1])
            {
                BladePowerStatePacket currState = ChassisState.BladePower[bladeId - 1].GetBladePowerState();

                if (currState.CompletionCode != CompletionCode.Success
                    || (currState.BladePowerState == (byte)Contracts.PowerState.OFF))
                {
                    // No return here, because we still want to return a BMC state on the fall through,
                    // if Blade enable read fails for whatever reason
                    Tracer.WriteWarning("PowerOn: Blade {0} Power Enable state read failed (Completion Code: {1:X})", bladeId, currState.CompletionCode);

                    bladePowerSwitchStatePacket = ChassisState.BladePower[bladeId - 1].SetBladePowerState((byte)PowerState.ON);
                    status = bladePowerSwitchStatePacket.CompletionCode;
                    Tracer.WriteInfo("Hard poweron status " + status);

                    if (status == CompletionCode.Success)
                    {
                        // Hard power on status is true, so Blade should be set to Initialization state on success
                        Tracer.WriteInfo("PowerOn: State Transition for blade {0}: {1} -> Initialization", bladeId,
                            ChassisState.GetStateName((byte)bladeId));

                        ChassisState.SetBladeState((byte)bladeId, (byte)BladeState.Initialization);
                        powerOnStatus = true;
                    }
                    else
                    {
                        Tracer.WriteWarning("PowerOn: Hard Power On failed for SledId {0} with code {1:X}", bladeId, status);
                    }
                }
                else
                {
                    powerOnStatus = true; // the blade was already powered on, so we dont power it on again
                }
            }
            return powerOnStatus;
        }
        /// <summary>
        /// Internal method to Power off blade
        /// </summary>
        /// <param name="bladeId">Blade ID(1-48)</param>
        /// <returns>true/false if operation was success/failure</returns>
        internal static bool PowerOff(int bladeId)
        {
            Tracer.WriteInfo("Received poweroff({0})", bladeId);
            bool powerOffStatus = false;

            BladePowerStatePacket bladePowerSwitchStatePacket = new BladePowerStatePacket();

            // Serialize power off and power on, on the same lock variable per blade, so we prevent inconsistent power state behavior
            lock (ChassisState.locker[bladeId - 1])
            {
                bladePowerSwitchStatePacket = ChassisState.BladePower[bladeId - 1].SetBladePowerState((byte)PowerState.OFF);
                CompletionCode status = bladePowerSwitchStatePacket.CompletionCode;

                // Sleep for specified amount of time after blade hard power off to prevent hardware inconsistent state
                // - hot-swap controller not completely draining its capacitance leading to inconsistent power state issues
                Thread.Sleep(ConfigLoaded.WaitTimeAfterBladeHardPowerOffInMsecs);

                Tracer.WriteInfo("PowerOff: Return: {0}", status);

                if (status != CompletionCode.Success)
                {
                    Tracer.WriteError("PowerOff: Blade Hard Power Off Failed with Completion code {0:X}", status);
                    powerOffStatus = false;
                }
                else
                {
                    powerOffStatus = true;
                    // set state to Hard Power Off
                    Tracer.WriteInfo("PowerOff: State Transition for blade {0}: {1} -> HardPowerOff", bladeId,
                        ChassisState.GetStateName((byte)bladeId));

                    ChassisState.SetBladeState((byte)bladeId, (byte)BladeState.HardPowerOff);
                    ChassisState.PowerFailCount[bladeId - 1] = 0;
                    // Clear blade type and cache
                    ChassisState.BladeTypeCache[bladeId - 1] = (byte)BladeType.Unknown;
                    WcsBladeFacade.ClearBladeClassification((byte)bladeId);
                }
            }
            return powerOffStatus;
        }
        /// <summary>
        /// Function that gets fan speed requirements
        /// from all blades. It also updates the blade states.
        /// </summary>
        private static void GetAllBladePwmRequirements()
        {
            // Rate is required to timestep over each individual Blade call
            double rate     = (double)getBladePwmReqtTimePeriodInMilliseconds / (double)MaxSledCount;
            double timeDiff = 0;

            for (byte blade = 1; blade <= MaxSledCount; blade++)
            {
                // Handle shutdown state
                if (ChassisState.ShutDown)
                {
                    return;
                }

                // default PWM setting
                byte PWM = (byte)ConfigLoaded.MinPWM;

                // Query blade type from IPMI layer
                ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification;

                // wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement

                double sleepTime = rate - timeDiff;

                if (sleepTime > rate)
                {
                    sleepTime = rate;
                }
                if (sleepTime > 0)
                {
                    Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime));
                }

                Tracer.WriteInfo("GetBladeRequirement called at {0} for BladeId {1} (state: {2})", DateTime.Now, blade,
                                 ChassisState.GetStateName(blade));

                // Check for the condition where known state is hardpoweroff, but someone plugged a new blade in
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff)
                {
                    CheckPowerEnableState(blade);
                }

                // Log Start time
                DateTime startTime = DateTime.Now;

                #region Check fail State -> Initialize

                // If blade was in Fail state
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail)
                {
                    // If failed count is greater than a maximum value, we move it to Initialization state
                    if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount)
                    {
                        // Move to Initialization state so that this blade could be reinitialized
                        Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Initialization", blade,
                                         ChassisState.GetStateName(blade));
                        ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                    }
                    else
                    {
                        // Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there.
                        // do not allow retries on Get System Guid
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for blade {0}, GUID: {1}", blade, guid.Guid.ToString());

                            DeviceGuid cachedGuid = WcsBladeFacade.GetCachedGuid(blade);

                            if (guid.Guid == cachedGuid.Guid)
                            {
                                // Change state to Probation and assume the system was in fail due to timeout.
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade,
                                                 ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Probation);
                            }
                            else
                            {
                                // Change state to Initialization as the device has changed.
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade,
                                                 ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                            }
                        }
                        else
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID returns a bad completion status: {0}", guid.CompletionCode);
                        }
                    }

                    // Increase time spent in Fail state everytime we are in this state
                    ChassisState.FailCount[blade - 1]++;
                }

                #endregion

                #region Move Initialize -> Probation

                // Handles Initialization
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization)
                {
                    BladePowerStatePacket powerstate = ChassisState.BladePower[blade - 1].GetCachedBladePowerState();

                    if (powerstate.CompletionCode == 0)
                    {
                        if (powerstate.DecompressionTime == 0)
                        {
                            // Will result in Hard Power off or Probation
                            ReInitialize(blade);
                        }
                    }
                }

                #endregion


                // Normal operation - possible states are probation or healthy
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation ||
                    ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy)
                {
                    #region Jbod (no sensor reading)

                    if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod)
                    {
                        // Do not allow retries on system guid.
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for JBOD {0}, GUID: {1}",
                                             blade, guid.Guid.ToString());

                            // Change state to Healthy
                            if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                            {
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Healthy",
                                                 blade, ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            }
                        }
                        else
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID for JBOD {0} failed with status {1}",
                                             blade, guid.CompletionCode);
                            // Set it to failed state, where we will retry guids and reinitialize if needed
                            Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Fail",
                                             blade, ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                        }

                        // No need to check for sensor reading, just continue
                        continue;
                    }

                    #endregion

                    #region Server -> Get PWM move to Healthy or move to Fail

                    // Call temperature reading list command
                    SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System);

                    if (Temps.CompletionCode != (byte)CompletionCode.Success)
                    {
                        Tracer.WriteWarning("GetAllBladePwmRequirements: BladeId: {0} - GetSensorReading for temperature failed with code {1:X}",
                                            blade, Temps.CompletionCode);

                        // Move to Fail state if no readings were obtained
                        Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Fail", blade,
                                         ChassisState.GetStateName(blade));

                        ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                    }
                    else
                    {
                        Tracer.WriteInfo("GetAllBladePwmRequirements: #### BladeId = " + blade + " Sensor id= " + ConfigLoaded.InputSensor +
                                         " Sensor reading = " + Temps.Reading + " Raw = " + Temps.RawReading +
                                         ", LowerNonCritical= " + ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold);

                        // Handle state logic if needed
                        // Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded
                        if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                        {
                            // Change state to healthy
                            Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Healthy",
                                             blade, ChassisState.GetStateName(blade));

                            ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            ChassisState.FailCount[blade - 1] = 0; // reset the fail count

                            // When a blade transitions to 'Healthy' state, enable/disable default blade operations
                            EnableDisableDefaultBladeOperations(blade);
                        }

                        if (ConfigLoaded.InputSensor != 1) // Non-PWM sensor.
                        {
                            PWM = GetPwmFromTemperature(Temps.Reading,
                                                        ConfigLoaded.SensorLowThreshold,
                                                        ConfigLoaded.SensorHighThreshold);
                        }
                        else
                        {
                            // PWM should never be higher or lower than the threshold.
                            if (Temps.Reading < ConfigLoaded.MinPWM || Temps.Reading > ConfigLoaded.MaxPWM)
                            {
                                Tracer.WriteWarning("PWM value " + Temps.Reading + " on blade " + blade +
                                                    " is out of range (lowThreshold: " + ConfigLoaded.MinPWM +
                                                    " - highThreshold: " + ConfigLoaded.MaxPWM);

                                PWM = (byte)ConfigLoaded.MinPWM;
                            }
                            else
                            {
                                PWM = (byte)Temps.Reading;
                            }
                        }

                        Tracer.WriteInfo("PWM value on blade {0} for Sensor {1} = {2}", blade, InputSensor, PWM);
                    }

                    #endregion
                }

                // write value into requirements table
                BladeRequirementTable[blade - 1] = PWM;

                // Log end time and capture time of execution for sensor get command
                DateTime endTime = DateTime.Now;
                timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds
            }
        }
        private BladePowerStatePacket SetBladePowerState(byte deviceId, byte state)
        {
            // Initialize return packet
            BladePowerStatePacket returnPacket = new BladePowerStatePacket();
            returnPacket.CompletionCode = CompletionCode.UnspecifiedError;
            returnPacket.BladePowerState = 3;

            Tracer.WriteInfo("SetSledPowerState Switch id: " + deviceId);

            if (state == (byte)Contracts.PowerState.ON)
            {
                try
                {
                    BladePowerStateOnResponse response = (BladePowerStateOnResponse)this.SendReceive(this.DeviceType, deviceId, new BladePowerStateOnRequest(state), typeof(BladePowerStateOnResponse));
                    if (response.CompletionCode != 0)
                    {
                        returnPacket.CompletionCode = (CompletionCode)response.CompletionCode;
                    }
                    else
                    {
                        returnPacket.CompletionCode = CompletionCode.Success;
                        returnPacket.BladePowerState = 1; //NOTE: response is not actually returned.
                        returnPacket.DecompressionTime = response.DecompressionTime;
                    }
                }
                catch (System.Exception ex)
                {
                    returnPacket.CompletionCode = CompletionCode.UnspecifiedError;
                    returnPacket.BladePowerState = 3;
                    Tracer.WriteError(this.DeviceId, DeviceType.Power, ex);
                }
            }
            else
            {
                try
                {
                    BladePowerStateOffResponse response = (BladePowerStateOffResponse)this.SendReceive(this.DeviceType, deviceId, new BladePowerStateOffRequest(state), typeof(BladePowerStateOffResponse));
                    if (response.CompletionCode != 0)
                    {
                        returnPacket.CompletionCode = (CompletionCode)response.CompletionCode;
                    }
                    else
                    {
                        returnPacket.CompletionCode = CompletionCode.Success;
                        returnPacket.BladePowerState = 0;  // blade is off
                        returnPacket.DecompressionTime = 0; // blade is off.
                    }
                }
                catch (System.Exception ex)
                {
                    returnPacket.CompletionCode = CompletionCode.UnspecifiedError;
                    returnPacket.BladePowerState = 3;
                    Tracer.WriteError(this.DeviceId, DeviceType.Power, ex);
                }
            }

            return returnPacket;
        }
        private BladePowerStatePacket GetCachedBladePowerState(byte deviceId)
        {
            // Initialize return packet
            BladePowerStatePacket returnPacket = new BladePowerStatePacket();
            returnPacket.CompletionCode = CompletionCode.UnspecifiedError;
            returnPacket.BladePowerState = 3;

            try
            {
                BladeCachedPowerStateResponse stateResponse = (BladeCachedPowerStateResponse)this.SendReceive(this.DeviceType, deviceId, new BladeCachedPowerStateRequest(), typeof(BladeCachedPowerStateResponse));
                if (stateResponse.CompletionCode != 0)
                {
                    returnPacket.CompletionCode = (CompletionCode)stateResponse.CompletionCode;
                }
                else
                {
                    returnPacket.CompletionCode = CompletionCode.Success;
                    returnPacket.BladePowerState = stateResponse.BladePowerState;
                    returnPacket.DecompressionTime = stateResponse.DecompressionTime;
                }
            }
            catch (System.Exception ex)
            {
                returnPacket.CompletionCode = CompletionCode.UnspecifiedError;
                returnPacket.BladePowerState = 3;
                Tracer.WriteError(this.DeviceId, DeviceType.Power, ex);
            }
            return returnPacket;
        }