/// <summary>
 /// Checks the state of the current device/blade id passed, return true if the device is not power off
 /// </summary>
 /// <param name="bladeId"></param>
 /// <returns></returns>
 public static bool checkBladeStateValidity(byte bladeId)
 {
     if (ChassisState.GetStateName(bladeId) == Enum.GetName(typeof(BladeState), BladeState.HardPowerOff))
     {
         return(false);
     }
     else
     {
         return(true);
     }
 }
Esempio n. 2
0
        /// <summary>
        /// Reinitialize the sled and set chassis state
        /// </summary>
        private void ReInitialize(byte sledId)
        {
            // Serialize initialize and power behavior per sled
            lock (ChassisState._lock[sledId - 1])
            {
                ChassisState.FailCount[sledId - 1] = 0;                // reset fail count since we are going to reinitialize the blade

                bool status = WcsBladeFacade.InitializeClient(sledId); // TODO: no completion code, only byte status returned

                if (status != true)
                {
                    // Initialization failed - move to fail state before retrying again
                    Tracer.WriteInfo("Reinitialization failed with code: {0} for Sled: {1}", status, sledId);
                    Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Fail", sledId,
                                     ChassisState.GetStateName(sledId));

                    ChassisState.SetBladeState((byte)sledId, (byte)BladeState.Fail);

                    // check power status to see if the blade was manually switched off or removed
                    BladePowerStatePacket response = ChassisState.BladePower[sledId - 1].GetBladePowerState();

                    // If the blade was turned off, set correct status / TODO: do we need this here?
                    if (response.BladePowerState == (byte)Contracts.PowerState.OFF)
                    {
                        Tracer.WriteInfo("SledId {0} is in hard power off state", sledId);
                        Tracer.WriteInfo("State Transition for Sled {0}: {1} -> HardPowerOff", sledId,
                                         ChassisState.GetStateName(sledId));

                        ChassisState.SetBladeState(sledId, (byte)BladeState.HardPowerOff);
                    }
                }
                else
                {
                    // State change: I -> P
                    Tracer.WriteInfo("Reinitialization of Sled: {0} succeeded with status {1}", sledId, status);

                    Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", sledId,
                                     ChassisState.GetStateName(sledId));
                    ChassisState.SetBladeState(sledId, (byte)BladeState.Probation);

                    // Initialize Blade Type (Type might have changed when Blades were reinserted)
                    if (WcsBladeFacade.clients.ContainsKey(sledId))
                    {
                        ChassisState.BladeTypeCache[sledId - 1] = (byte)WcsBladeFacade.clients[sledId].BladeClassification;
                    }
                    else
                    {
                        ChassisState.BladeTypeCache[sledId - 1] = (byte)BladeType.Unknown;
                    }
                }
            }
        }
Esempio n. 3
0
        /// <summary>
        /// Internal operation to call both hard power on (soft power on is not exposed to the user)
        /// </summary>
        /// <param name="bladeId">Blade ID</param>
        /// <returns>True/false for success/failure</returns>
        internal static bool PowerOn(int bladeId)
        {
            Tracer.WriteInfo("Received poweron({0})", bladeId);
            bool powerOnStatus = false;

            BladePowerStatePacket bladePowerSwitchStatePacket = new BladePowerStatePacket();
            CompletionCode        status;

            // Hard Power enable
            // Serialize setting of state and actual code logic
            lock (ChassisState.locker[bladeId - 1])
            {
                BladePowerStatePacket currState = ChassisState.BladePower[bladeId - 1].GetBladePowerState();

                if (currState.CompletionCode != CompletionCode.Success ||
                    (currState.BladePowerState == (byte)Contracts.PowerState.OFF))
                {
                    // No return here, because we still want to return a BMC state on the fall through,
                    // if Blade enable read fails for whatever reason
                    Tracer.WriteWarning("PowerOn: Blade {0} Power Enable state read failed (Completion Code: {1:X})", bladeId, currState.CompletionCode);

                    bladePowerSwitchStatePacket = ChassisState.BladePower[bladeId - 1].SetBladePowerState((byte)PowerState.ON);
                    status = bladePowerSwitchStatePacket.CompletionCode;
                    Tracer.WriteInfo("Hard poweron status " + status);

                    if (status == CompletionCode.Success)
                    {
                        // Hard power on status is true, so Blade should be set to Initialization state on success
                        Tracer.WriteInfo("PowerOn: State Transition for blade {0}: {1} -> Initialization", bladeId,
                                         ChassisState.GetStateName((byte)bladeId));

                        ChassisState.SetBladeState((byte)bladeId, (byte)BladeState.Initialization);
                        powerOnStatus = true;
                    }
                    else
                    {
                        Tracer.WriteWarning("PowerOn: Hard Power On failed for BladeId {0} with code {1:X}", bladeId, status);
                    }
                }
                else
                {
                    powerOnStatus = true; // the blade was already powered on, so we dont power it on again
                }
            }
            return(powerOnStatus);
        }
        /// <summary>
        /// Checks the power enable state of the blade and changes state accordingly
        /// </summary>
        /// <param name="deviceId"></param>
        private static void CheckPowerEnableState(byte deviceId)
        {
            // Serialize power behavior
            lock (ChassisState.locker[deviceId - 1])
            {
                BladePowerStatePacket response = ChassisState.BladePower[deviceId - 1].GetCachedBladePowerState();

                if (response.CompletionCode != CompletionCode.Success)
                {
                    Tracer.WriteInfo("CheckPowerEnableState: Blade {0} Power Enable state read failed (Completion Code: {1:X})",
                                     deviceId, response.CompletionCode);
                }
                else
                {
                    if (response.BladePowerState == (byte)Contracts.PowerState.ON)
                    {
                        if (ChassisState.GetBladeState((byte)deviceId) == (byte)BladeState.HardPowerOff)
                        {
                            // Blade is powered on, move to initialization state
                            Tracer.WriteInfo("CheckPowerEnableState: State Transition for blade {0}: {1} -> Initialization",
                                             deviceId, ChassisState.GetStateName(deviceId));
                            ChassisState.SetBladeState((byte)deviceId, (byte)BladeState.Initialization);
                        }
                    }
                    else if (response.BladePowerState == (byte)Contracts.PowerState.OFF)
                    {
                        if (ChassisState.GetBladeState((byte)deviceId) != (byte)BladeState.HardPowerOff)
                        {
                            // Blade is powered off, move to PowerOff state
                            Tracer.WriteInfo("CheckPowerEnableState: State Transition for blade {0}: {1} -> HardPowerOff",
                                             deviceId, ChassisState.GetStateName(deviceId));

                            ChassisState.SetBladeState((byte)deviceId, (byte)BladeState.HardPowerOff);
                        }
                    }
                    else
                    {
                        Tracer.WriteInfo("CheckPowerEnableState: Getting out of else block");
                        // TODO: do we need to do anything for state that is NA
                    }
                }
            }
        }
        /// <summary>
        /// Initialize blades state (powered?) and type (compute/JBOD)
        /// </summary>
        private static void BladeInitialize()
        {
            // Get power status of enable pin for each blade and update blade state
            for (byte deviceId = 1; deviceId <= MaxSledCount; deviceId++)
            {
                CheckPowerEnableState(deviceId);
            }

            // Initialize Wcs Blade - TODO: This initialize should return some status
            WcsBladeFacade.Initialize();  // This method just creates IPMI Client Class for each blade.

            Tracer.WriteInfo("BladeInitialize: IPMI Facade Initialized, Number of blades initialized: {0}", WcsBladeFacade.Initialized);

            // check all client initialization status and update state
            Tracer.WriteInfo("BladeInitialize: Checking client status for {0} blades", MaxSledCount);
            for (byte deviceId = 1; deviceId <= MaxSledCount; deviceId++)
            {
                // TODO: How to check initialized status, now that this has become a function
                if (WcsBladeFacade.clients[deviceId].Initialize()) // This method logs on to an IPMI session.
                {
                    // If initialized is true, change state to probation
                    Tracer.WriteInfo("BladeInitialize: State Transition for blade {0}: {1} -> Probation",
                                     deviceId, ChassisState.GetStateName(deviceId));

                    ChassisState.SetBladeState(deviceId, (byte)BladeState.Probation);
                }
                else
                {
                    Tracer.WriteInfo("BladeInitialize: Blade not initialized: Blade {0}", deviceId);
                }
            }

            if (WcsBladeFacade.Initialized > 0)
            {
                // Identify what kind of sleds these are
                for (byte loop = 1; loop <= MaxSledCount; loop++)
                {
                    byte deviceId = WcsBladeFacade.clients[loop].DeviceId;
                    ChassisState.BladeTypeCache[deviceId - 1] = (byte)WcsBladeFacade.clients[loop].BladeClassification;
                }
            }
        }
Esempio n. 6
0
        /// <summary>
        /// Internal method to Power off blade
        /// </summary>
        /// <param name="bladeId">Blade ID(1-48)</param>
        /// <returns>true/false if operation was success/failure</returns>
        internal static bool PowerOff(int bladeId)
        {
            Tracer.WriteInfo("Received poweroff({0})", bladeId);
            bool powerOffStatus = false;

            BladePowerStatePacket bladePowerSwitchStatePacket = new BladePowerStatePacket();

            // Serialize power off and power on, on the same lock variable per blade, so we prevent inconsistent power state behavior
            lock (ChassisState.locker[bladeId - 1])
            {
                bladePowerSwitchStatePacket = ChassisState.BladePower[bladeId - 1].SetBladePowerState((byte)PowerState.OFF);
                CompletionCode status = bladePowerSwitchStatePacket.CompletionCode;

                // Sleep for specified amount of time after blade hard power off to prevent hardware inconsistent state
                // - hot-swap controller not completely draining its capacitance leading to inconsistent power state issues
                Thread.Sleep(ConfigLoaded.WaitTimeAfterBladeHardPowerOffInMsecs);

                Tracer.WriteInfo("PowerOff: Return: {0}", status);

                if (status != CompletionCode.Success)
                {
                    Tracer.WriteError("PowerOff: Blade Hard Power Off Failed with Completion code {0:X}", status);
                    powerOffStatus = false;
                }
                else
                {
                    powerOffStatus = true;
                    // set state to Hard Power Off
                    Tracer.WriteInfo("PowerOff: State Transition for blade {0}: {1} -> HardPowerOff", bladeId,
                                     ChassisState.GetStateName((byte)bladeId));

                    ChassisState.SetBladeState((byte)bladeId, (byte)BladeState.HardPowerOff);
                    ChassisState.PowerFailCount[bladeId - 1] = 0;
                    // Clear blade type and cache
                    ChassisState.BladeTypeCache[bladeId - 1] = (byte)BladeType.Unknown;
                    WcsBladeFacade.ClearBladeClassification((byte)bladeId);
                }
            }
            return(powerOffStatus);
        }
        /// <summary>
        /// Function that gets fan speed requirements
        /// from all blades. It also updates the blade states.
        /// </summary>
        private static void GetAllBladePwmRequirements()
        {
            // Rate is required to timestep over each individual Blade call
            double rate     = (double)getBladePwmReqtTimePeriodInMilliseconds / (double)MaxSledCount;
            double timeDiff = 0;

            for (byte blade = 1; blade <= MaxSledCount; blade++)
            {
                // Handle shutdown state
                if (ChassisState.ShutDown)
                {
                    return;
                }

                // default PWM setting
                byte PWM = (byte)ConfigLoaded.MinPWM;

                // Query blade type from IPMI layer
                ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification;

                // wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement

                double sleepTime = rate - timeDiff;

                if (sleepTime > rate)
                {
                    sleepTime = rate;
                }
                if (sleepTime > 0)
                {
                    Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime));
                }

                Tracer.WriteInfo("GetBladeRequirement called at {0} for BladeId {1} (state: {2})", DateTime.Now, blade,
                                 ChassisState.GetStateName(blade));

                // Check for the condition where known state is hardpoweroff, but someone plugged a new blade in
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff)
                {
                    CheckPowerEnableState(blade);
                }

                // Log Start time
                DateTime startTime = DateTime.Now;

                #region Check fail State -> Initialize

                // If blade was in Fail state
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail)
                {
                    // If failed count is greater than a maximum value, we move it to Initialization state
                    if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount)
                    {
                        // Move to Initialization state so that this blade could be reinitialized
                        Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Initialization", blade,
                                         ChassisState.GetStateName(blade));
                        ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                    }
                    else
                    {
                        // Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there.
                        // do not allow retries on Get System Guid
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for blade {0}, GUID: {1}", blade, guid.Guid.ToString());

                            DeviceGuid cachedGuid = WcsBladeFacade.GetCachedGuid(blade);

                            if (guid.Guid == cachedGuid.Guid)
                            {
                                // Change state to Probation and assume the system was in fail due to timeout.
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade,
                                                 ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Probation);
                            }
                            else
                            {
                                // Change state to Initialization as the device has changed.
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade,
                                                 ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                            }
                        }
                        else
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID returns a bad completion status: {0}", guid.CompletionCode);
                        }
                    }

                    // Increase time spent in Fail state everytime we are in this state
                    ChassisState.FailCount[blade - 1]++;
                }

                #endregion

                #region Move Initialize -> Probation

                // Handles Initialization
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization)
                {
                    BladePowerStatePacket powerstate = ChassisState.BladePower[blade - 1].GetCachedBladePowerState();

                    if (powerstate.CompletionCode == 0)
                    {
                        if (powerstate.DecompressionTime == 0)
                        {
                            // Will result in Hard Power off or Probation
                            ReInitialize(blade);
                        }
                    }
                }

                #endregion


                // Normal operation - possible states are probation or healthy
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation ||
                    ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy)
                {
                    #region Jbod (no sensor reading)

                    if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod)
                    {
                        // Do not allow retries on system guid.
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for JBOD {0}, GUID: {1}",
                                             blade, guid.Guid.ToString());

                            // Change state to Healthy
                            if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                            {
                                Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Healthy",
                                                 blade, ChassisState.GetStateName(blade));
                                ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            }
                        }
                        else
                        {
                            Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID for JBOD {0} failed with status {1}",
                                             blade, guid.CompletionCode);
                            // Set it to failed state, where we will retry guids and reinitialize if needed
                            Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Fail",
                                             blade, ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                        }

                        // No need to check for sensor reading, just continue
                        continue;
                    }

                    #endregion

                    #region Server -> Get PWM move to Healthy or move to Fail

                    // Call temperature reading list command
                    SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System);

                    if (Temps.CompletionCode != (byte)CompletionCode.Success)
                    {
                        Tracer.WriteWarning("GetAllBladePwmRequirements: BladeId: {0} - GetSensorReading for temperature failed with code {1:X}",
                                            blade, Temps.CompletionCode);

                        // Move to Fail state if no readings were obtained
                        Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Fail", blade,
                                         ChassisState.GetStateName(blade));

                        ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                    }
                    else
                    {
                        Tracer.WriteInfo("GetAllBladePwmRequirements: #### BladeId = " + blade + " Sensor id= " + ConfigLoaded.InputSensor +
                                         " Sensor reading = " + Temps.Reading + " Raw = " + Temps.RawReading +
                                         ", LowerNonCritical= " + ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold);

                        // Handle state logic if needed
                        // Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded
                        if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                        {
                            // Change state to healthy
                            Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Healthy",
                                             blade, ChassisState.GetStateName(blade));

                            ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            ChassisState.FailCount[blade - 1] = 0; // reset the fail count

                            // When a blade transitions to 'Healthy' state, enable/disable default blade operations
                            EnableDisableDefaultBladeOperations(blade);
                        }

                        if (ConfigLoaded.InputSensor != 1) // Non-PWM sensor.
                        {
                            PWM = GetPwmFromTemperature(Temps.Reading,
                                                        ConfigLoaded.SensorLowThreshold,
                                                        ConfigLoaded.SensorHighThreshold);
                        }
                        else
                        {
                            // PWM should never be higher or lower than the threshold.
                            if (Temps.Reading < ConfigLoaded.MinPWM || Temps.Reading > ConfigLoaded.MaxPWM)
                            {
                                Tracer.WriteWarning("PWM value " + Temps.Reading + " on blade " + blade +
                                                    " is out of range (lowThreshold: " + ConfigLoaded.MinPWM +
                                                    " - highThreshold: " + ConfigLoaded.MaxPWM);

                                PWM = (byte)ConfigLoaded.MinPWM;
                            }
                            else
                            {
                                PWM = (byte)Temps.Reading;
                            }
                        }

                        Tracer.WriteInfo("PWM value on blade {0} for Sensor {1} = {2}", blade, InputSensor, PWM);
                    }

                    #endregion
                }

                // write value into requirements table
                BladeRequirementTable[blade - 1] = PWM;

                // Log end time and capture time of execution for sensor get command
                DateTime endTime = DateTime.Now;
                timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds
            }
        }
Esempio n. 8
0
        /// <summary>
        /// Function that gets all the fan speed requirements
        /// from the Blade.  It also updates the balde state
        /// </summary>
        private void GetAllBladePwmRequirements()
        {
            // Rate is required to timestep over each individual Blade call
            double rate     = (double)GetTimePeriod / (double)MaxSledCount;
            double timeDiff = 0;

            for (byte blade = 1; blade <= MaxSledCount; blade++)
            {
                // Handle shutdown state
                if (ChassisState.ShutDown)
                {
                    return;
                }

                // default PWM setting
                byte PWM = (byte)ConfigLoaded.MinPWM;

                // Query blade type from IPMI layer
                ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification;

                // wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement

                double sleepTime = rate - timeDiff;

                if (sleepTime > rate)
                {
                    sleepTime = rate;
                }
                if (sleepTime > 0)
                {
                    Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime));
                }
                if (CommunicationDevice.IsSafeMode())
                {
                    // Do not perform any sensor reading - continue in the for loop
                    Tracer.WriteInfo("Monitoring thread: Safe Mode, Skipping sensor read");
                    continue;
                }
                Tracer.WriteInfo("GetBladeRequirement called at {0} for sledId {1} (state: {2})", DateTime.Now, blade,
                                 ChassisState.GetStateName(blade));

                // Check for the condition where known state is hardpoweroff, but someone plugged a new blade in
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff)
                {
                    ChassisState.PowerFailCount[blade - 1]++;
                    // TODO: identify if this period is sufficient to do this check
                    if (ChassisState.PowerFailCount[blade - 1] > (ConfigLoaded.MaxRetries * ConfigLoaded.Population))
                    {
                        CheckPowerEnableState(blade);
                        ChassisState.PowerFailCount[blade - 1] = 0;
                    }
                }
                // Log Start time
                DateTime startTime = DateTime.Now;

                // If blade was in Fail state
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail)
                {
                    // If failed count is greater than a maximum value, we move it to Initialization state

                    if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount)
                    {
                        // Move to Initialization state so that this sled could be reinitialized
                        Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Initialization", blade,
                                         ChassisState.GetStateName(blade));
                        ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
                    }
                    else
                    {
                        // Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GUID present for sled {0}, GUID: {1}", blade, guid.Guid.ToString());

                            // Change state to Probation
                            Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", blade,
                                             ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Probation);
                        }
                        else
                        {
                            Tracer.WriteInfo("Get System GUID returns a bad completion status: {0}", guid.CompletionCode);
                        }
                    }
                    // Increase time spent in Fail state everytime we are in this state
                    ChassisState.FailCount[blade - 1]++;
                }

                // Handles Initialization
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization)
                {
                    this.ReInitialize(blade);
                }

                // Normal operation - possible states are probation or healthy
                if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation ||
                    ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy)
                {
                    if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod)
                    {
                        DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade);
                        if (guid.CompletionCode == (byte)CompletionCode.Success)
                        {
                            Tracer.WriteInfo("GUID present for jbod {0}, GUID: {1}", blade, guid.Guid.ToString());

                            // Change state to Probation
                            Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Healthy", blade, ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                        }
                        else
                        {
                            Tracer.WriteInfo("Get System GUID for jbod {0} failed with status {1}", blade, guid.CompletionCode);
                            // Set it to failed state, where we will retry guids and reinitialize if needed
                            Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Fail", blade, ChassisState.GetStateName(blade));
                            ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                        }

                        // No need to check for sensor reading, just continue
                        continue;
                    }
                    // Call temperature reading list command
                    SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System);

                    if (Temps.CompletionCode != (byte)CompletionCode.Success)
                    {
                        Tracer.WriteWarning("SledId: {0} - getTempSensorReading failed with code {1:X}", blade, Temps.CompletionCode);

                        // Move to Fail state if no readings were obtained
                        Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Fail", blade,
                                         ChassisState.GetStateName(blade));

                        ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
                    }
                    else
                    {
                        Tracer.WriteInfo("#### Sledid= " + blade + " Sensor id= " + ConfigLoaded.InputSensor + " Sensor reading= " +
                                         Temps.Reading + " Raw= " + Temps.RawReading + ", LowerNonCritical= " +
                                         ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold);

                        // Handle state logic if needed
                        // Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded
                        if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
                        {
                            // Change state to healthy
                            Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Healthy", blade,
                                             ChassisState.GetStateName(blade));

                            ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
                            ChassisState.FailCount[blade - 1] = 0; // reset the fail count
                        }

                        PWM = GetPwmFromTemperature(Temps.Reading,
                                                    ConfigLoaded.SensorLowThreshold,
                                                    ConfigLoaded.SensorHighThreshold);

                        Tracer.WriteInfo("PWM value for Sensor {0} = {1}", InputSensor, PWM);
                    }
                }

                // write value into requirements table
                BladeRequirementTable[blade - 1] = PWM;

                // Log end time and capture time of execution for sensor get command
                DateTime endTime = DateTime.Now;
                timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds
            }
        }
Esempio n. 9
0
        /// <summary>
        /// Initialize Chassis constants and configs
        /// </summary>
        internal byte Initialize()
        {
            Tracer.WriteInfo("Initializing state");

            byte status = (byte)CompletionCode.UnspecifiedError;

            ChassisState.Initialize();

            Tracer.WriteInfo("Initializing Communication Device");
            // Initializer lower layer communication device
            CompletionCode completionCode = CommunicationDevice.Init();

            if (CompletionCodeChecker.Failed(completionCode))
            {
                Tracer.WriteWarning("Initialization failed: {0}", completionCode);
                int loop = 0;

                // Retry 3 times before failing completely
                for (loop = 0; loop < ConfigLoaded.MaxRetries; loop++)
                {
                    Tracer.WriteInfo("Initialization Retry: {0}", loop);

                    completionCode = CommunicationDevice.Init();
                    if (CompletionCodeChecker.Succeeded(completionCode))
                    {
                        break;
                    }
                }

                if (loop == ConfigLoaded.MaxRetries)
                {
                    Tracer.WriteError("Re-attempt at Communication Device Initialization failed with code: {0}", completionCode);
                    return(status);
                }
            }
            if (CompletionCodeChecker.Succeeded(completionCode))
            {
                Tracer.WriteInfo("Communication Device Initialized");
                status = (byte)CompletionCode.Success;
            }

            // Get power status of enable pin for each blade and update blade state
            for (byte deviceId = 1; deviceId <= MaxSledCount; deviceId++)
            {
                CheckPowerEnableState(deviceId);
            }

            // Initialize Wcs Blade - TODO: This initialize should return some status
            WcsBladeFacade.Initialize();  // This method just creates IPMI Client Class for each blade.
            Tracer.WriteInfo("IPMI Facade Initialized, Number of blades initialized: {0}", WcsBladeFacade.Initialized);

            // check all client initialization status and update state
            Tracer.WriteInfo("Checking client status for {0} blades", MaxSledCount);
            for (byte deviceId = 1; deviceId <= MaxSledCount; deviceId++)
            {
                // TODO: How to check initialized status, now that this has become a function
                if (WcsBladeFacade.clients[deviceId].Initialize()) // This method logs on to an IPMI session.
                {
                    // If initialized is true, change state to probation
                    Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", deviceId,
                                     ChassisState.GetStateName(deviceId));

                    ChassisState.SetBladeState(deviceId, (byte)BladeState.Probation);
                }
                else
                {
                    Tracer.WriteInfo("Blade not initialized: Blade ", +deviceId);
                }
            }

            Tracer.WriteInfo("Initializing Watchdog Timer");

            // Initialize WatchDog Timer
            ChassisState.Wdt.EnableWatchDogTimer();

            Tracer.WriteInfo("Watchdog timer initialized");

            // Initialize internal chassis manager tables
            this.ChassisInternalInitialize();

            return(status);
        }