/// <summary> /// Checks the validity of the current blade id passed /// </summary> /// <param name="bladeId"></param> /// <returns></returns> public static bool checkBladeTypeValidity(byte bladeId) { try { // Open the request message using an xml reader XmlReader xr = OperationContext.Current.IncomingMessageHeaders.GetReaderAtHeader(0); // Split the URL at the API name--Parameters junction indicated by the '?' character - taking the first string will ignore all parameters string[] urlSplit = xr.ReadElementContentAsString().Split('/'); // Extract just the API name and rest of the URL, which will be the last item in the split using '/' string[] apiSplit = urlSplit[3].Split('?'); BladeType val = BladeType.Unknown; invalidBladeFunction.TryGetValue(apiSplit[0], out val); // If the blade type does not support this function, return false, so we can send back useful info to user if ((byte)val == ChassisState.GetBladeType(bladeId)) { Tracer.WriteWarning("Command {0} not valid for Blade id {1}, Blade Type {2}", apiSplit[0], bladeId, ChassisState.GetBladeTypeName((byte)val)); return(false); } } catch (Exception ex) { Tracer.WriteError("Checking Blade Type validity encountered an exception" + ex); // We decide to go ahead and issue the command to the blade if the blade type check fails with exception, // This is done in order to not penalize a user command based on some failure in checking // The command might fail eventually, but with an unhelpful error message } return(true); }
private static void EnableDisableDefaultBladeOperations(int bladeId) { // TODO: Check blade type etc and Kill any serial session // TODO: Add trace log messages // Check to see if the blade is hard powered off BladePowerStatePacket response = ChassisState.BladePower[bladeId - 1].GetBladePowerState(); if (response.CompletionCode != CompletionCode.Success) { // Log error here, and proceed to check blade state since we still want to check BMC soft power status // even if blade enable read failed for whatever reason Tracer.WriteError("EnableDisableDefaultBladeOperations: Blade {0} Power Enable state read failed (Completion Code: {1:X})", bladeId, response.CompletionCode); } else if (response.BladePowerState == (byte)Contracts.PowerState.OFF) { // If blade is hard powered off, no further processing is necessary return; } // If the blade is a Jbod, return since the operations done in this method do not apply for Jbods if (ChassisState.GetBladeType((byte)bladeId) == (byte)BladeType.Jbod) { Tracer.WriteInfo("EnableDisableDefaultBladeOperations (Blade#{0}): Ignoring since it is a Jbod", bladeId); return; } DatasafeOperationSupport.ProcessDatasafeAction(bladeId, ConfigLoaded.DatasafeOperationsEnabled ? DatasafeActions.EnableDatasafe : DatasafeActions.DisableDatasafe); if (ConfigLoaded.PsuAlertMonitorEnabled) { WcsBladeFacade.ActivatePsuAlert((byte)bladeId, true, BmcPsuAlertAction.ProcHotAndDpc, true); } else { WcsBladeFacade.ActivatePsuAlert((byte)bladeId, false, BmcPsuAlertAction.NoAction, true); } }
/// <summary> /// Function that gets fan speed requirements /// from all blades. It also updates the blade states. /// </summary> private static void GetAllBladePwmRequirements() { // Rate is required to timestep over each individual Blade call double rate = (double)getBladePwmReqtTimePeriodInMilliseconds / (double)MaxSledCount; double timeDiff = 0; for (byte blade = 1; blade <= MaxSledCount; blade++) { // Handle shutdown state if (ChassisState.ShutDown) { return; } // default PWM setting byte PWM = (byte)ConfigLoaded.MinPWM; // Query blade type from IPMI layer ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification; // wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement double sleepTime = rate - timeDiff; if (sleepTime > rate) { sleepTime = rate; } if (sleepTime > 0) { Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime)); } Tracer.WriteInfo("GetBladeRequirement called at {0} for BladeId {1} (state: {2})", DateTime.Now, blade, ChassisState.GetStateName(blade)); // Check for the condition where known state is hardpoweroff, but someone plugged a new blade in if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff) { CheckPowerEnableState(blade); } // Log Start time DateTime startTime = DateTime.Now; #region Check fail State -> Initialize // If blade was in Fail state if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail) { // If failed count is greater than a maximum value, we move it to Initialization state if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount) { // Move to Initialization state so that this blade could be reinitialized Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Initialization", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Initialization); } else { // Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there. // do not allow retries on Get System Guid DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false); if (guid.CompletionCode == (byte)CompletionCode.Success) { Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for blade {0}, GUID: {1}", blade, guid.Guid.ToString()); DeviceGuid cachedGuid = WcsBladeFacade.GetCachedGuid(blade); if (guid.Guid == cachedGuid.Guid) { // Change state to Probation and assume the system was in fail due to timeout. Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Probation); } else { // Change state to Initialization as the device has changed. Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Probation", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Initialization); } } else { Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID returns a bad completion status: {0}", guid.CompletionCode); } } // Increase time spent in Fail state everytime we are in this state ChassisState.FailCount[blade - 1]++; } #endregion #region Move Initialize -> Probation // Handles Initialization if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization) { BladePowerStatePacket powerstate = ChassisState.BladePower[blade - 1].GetCachedBladePowerState(); if (powerstate.CompletionCode == 0) { if (powerstate.DecompressionTime == 0) { // Will result in Hard Power off or Probation ReInitialize(blade); } } } #endregion // Normal operation - possible states are probation or healthy if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation || ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy) { #region Jbod (no sensor reading) if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod) { // Do not allow retries on system guid. DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade, false); if (guid.CompletionCode == (byte)CompletionCode.Success) { Tracer.WriteInfo("GetAllBladePwmRequirements: GUID present for JBOD {0}, GUID: {1}", blade, guid.Guid.ToString()); // Change state to Healthy if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation) { Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Healthy", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Healthy); } } else { Tracer.WriteInfo("GetAllBladePwmRequirements: Get System GUID for JBOD {0} failed with status {1}", blade, guid.CompletionCode); // Set it to failed state, where we will retry guids and reinitialize if needed Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for JBOD {0}: {1} -> Fail", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Fail); } // No need to check for sensor reading, just continue continue; } #endregion #region Server -> Get PWM move to Healthy or move to Fail // Call temperature reading list command SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System); if (Temps.CompletionCode != (byte)CompletionCode.Success) { Tracer.WriteWarning("GetAllBladePwmRequirements: BladeId: {0} - GetSensorReading for temperature failed with code {1:X}", blade, Temps.CompletionCode); // Move to Fail state if no readings were obtained Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Fail", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Fail); } else { Tracer.WriteInfo("GetAllBladePwmRequirements: #### BladeId = " + blade + " Sensor id= " + ConfigLoaded.InputSensor + " Sensor reading = " + Temps.Reading + " Raw = " + Temps.RawReading + ", LowerNonCritical= " + ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold); // Handle state logic if needed // Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation) { // Change state to healthy Tracer.WriteInfo("GetAllBladePwmRequirements: State Transition for blade {0}: {1} -> Healthy", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Healthy); ChassisState.FailCount[blade - 1] = 0; // reset the fail count // When a blade transitions to 'Healthy' state, enable/disable default blade operations EnableDisableDefaultBladeOperations(blade); } if (ConfigLoaded.InputSensor != 1) // Non-PWM sensor. { PWM = GetPwmFromTemperature(Temps.Reading, ConfigLoaded.SensorLowThreshold, ConfigLoaded.SensorHighThreshold); } else { // PWM should never be higher or lower than the threshold. if (Temps.Reading < ConfigLoaded.MinPWM || Temps.Reading > ConfigLoaded.MaxPWM) { Tracer.WriteWarning("PWM value " + Temps.Reading + " on blade " + blade + " is out of range (lowThreshold: " + ConfigLoaded.MinPWM + " - highThreshold: " + ConfigLoaded.MaxPWM); PWM = (byte)ConfigLoaded.MinPWM; } else { PWM = (byte)Temps.Reading; } } Tracer.WriteInfo("PWM value on blade {0} for Sensor {1} = {2}", blade, InputSensor, PWM); } #endregion } // write value into requirements table BladeRequirementTable[blade - 1] = PWM; // Log end time and capture time of execution for sensor get command DateTime endTime = DateTime.Now; timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds } }
/// <summary> /// Function that gets all the fan speed requirements /// from the Blade. It also updates the balde state /// </summary> private void GetAllBladePwmRequirements() { // Rate is required to timestep over each individual Blade call double rate = (double)GetTimePeriod / (double)MaxSledCount; double timeDiff = 0; for (byte blade = 1; blade <= MaxSledCount; blade++) { // Handle shutdown state if (ChassisState.ShutDown) { return; } // default PWM setting byte PWM = (byte)ConfigLoaded.MinPWM; // Query blade type from IPMI layer ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification; // wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement double sleepTime = rate - timeDiff; if (sleepTime > rate) { sleepTime = rate; } if (sleepTime > 0) { Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime)); } if (CommunicationDevice.IsSafeMode()) { // Do not perform any sensor reading - continue in the for loop Tracer.WriteInfo("Monitoring thread: Safe Mode, Skipping sensor read"); continue; } Tracer.WriteInfo("GetBladeRequirement called at {0} for sledId {1} (state: {2})", DateTime.Now, blade, ChassisState.GetStateName(blade)); // Check for the condition where known state is hardpoweroff, but someone plugged a new blade in if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff) { ChassisState.PowerFailCount[blade - 1]++; // TODO: identify if this period is sufficient to do this check if (ChassisState.PowerFailCount[blade - 1] > (ConfigLoaded.MaxRetries * ConfigLoaded.Population)) { CheckPowerEnableState(blade); ChassisState.PowerFailCount[blade - 1] = 0; } } // Log Start time DateTime startTime = DateTime.Now; // If blade was in Fail state if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail) { // If failed count is greater than a maximum value, we move it to Initialization state if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount) { // Move to Initialization state so that this sled could be reinitialized Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Initialization", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Initialization); } else { // Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade); if (guid.CompletionCode == (byte)CompletionCode.Success) { Tracer.WriteInfo("GUID present for sled {0}, GUID: {1}", blade, guid.Guid.ToString()); // Change state to Probation Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Probation); } else { Tracer.WriteInfo("Get System GUID returns a bad completion status: {0}", guid.CompletionCode); } } // Increase time spent in Fail state everytime we are in this state ChassisState.FailCount[blade - 1]++; } // Handles Initialization if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization) { this.ReInitialize(blade); } // Normal operation - possible states are probation or healthy if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation || ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy) { if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod) { DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade); if (guid.CompletionCode == (byte)CompletionCode.Success) { Tracer.WriteInfo("GUID present for jbod {0}, GUID: {1}", blade, guid.Guid.ToString()); // Change state to Probation Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Healthy", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Healthy); } else { Tracer.WriteInfo("Get System GUID for jbod {0} failed with status {1}", blade, guid.CompletionCode); // Set it to failed state, where we will retry guids and reinitialize if needed Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Fail", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Fail); } // No need to check for sensor reading, just continue continue; } // Call temperature reading list command SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System); if (Temps.CompletionCode != (byte)CompletionCode.Success) { Tracer.WriteWarning("SledId: {0} - getTempSensorReading failed with code {1:X}", blade, Temps.CompletionCode); // Move to Fail state if no readings were obtained Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Fail", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Fail); } else { Tracer.WriteInfo("#### Sledid= " + blade + " Sensor id= " + ConfigLoaded.InputSensor + " Sensor reading= " + Temps.Reading + " Raw= " + Temps.RawReading + ", LowerNonCritical= " + ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold); // Handle state logic if needed // Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation) { // Change state to healthy Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Healthy", blade, ChassisState.GetStateName(blade)); ChassisState.SetBladeState(blade, (byte)BladeState.Healthy); ChassisState.FailCount[blade - 1] = 0; // reset the fail count } PWM = GetPwmFromTemperature(Temps.Reading, ConfigLoaded.SensorLowThreshold, ConfigLoaded.SensorHighThreshold); Tracer.WriteInfo("PWM value for Sensor {0} = {1}", InputSensor, PWM); } } // write value into requirements table BladeRequirementTable[blade - 1] = PWM; // Log end time and capture time of execution for sensor get command DateTime endTime = DateTime.Now; timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds } }