/// <summary>
/// Identifies the PSU vendor at each psu slot using the modelnumber API of the PsuBase class
/// (Assumes all PSU vendors implement the MFR_MODEL Pmbus command)
/// Based on the model number, we bind the Psu class object to the corresponding child (vendor) class object
/// </summary>
private static void PsuInitialize()
{
for (uint psuIndex = 0; psuIndex < ConfigLoaded.NumPsus; psuIndex++)
{
// Initially create instance of the base class
// Later.. based on the psu model number, we create the appropriate psu class object
Psu[psuIndex] = new PsuBase((byte)(psuIndex + 1));
PsuModelNumberPacket modelNumberPacket = new PsuModelNumberPacket();
modelNumberPacket = Psu[psuIndex].GetPsuModel();
string psuModelNumber = modelNumberPacket.ModelNumber;
PsuModel model = ConvertPsuModelNumberToPsuModel(psuModelNumber);
switch (model)
{
case PsuModel.Delta:
Psu[psuIndex] = new DeltaPsu((byte)(psuIndex + 1));
Tracer.WriteInfo("Delta PSU identified at slot-{0}, Model Number: {1}", psuIndex + 1, psuModelNumber);
break;
case PsuModel.EmersonWithLes:
Psu[psuIndex] = new EmersonPsu((byte)(psuIndex + 1), true);
Tracer.WriteInfo("Emerson with LES PSU identified at slot-{0}, Model Number: {1}", psuIndex + 1, psuModelNumber);
break;
case PsuModel.EmersonNonLes:
Psu[psuIndex] = new EmersonPsu((byte)(psuIndex + 1), false);
Tracer.WriteInfo("Emerson non-LES PSU identified at slot-{0}, Model Number: {1}", psuIndex + 1, psuModelNumber);
break;
default:
// Unknown PSUs
if (ConfigLoaded.ForceEmersonPsu)
{
// Force to Emerson PSU without LES to enable FW update methods in the EmersonPsu class to be called.
// This is useful when a previous FW update did not complete successfully.
// The PSU stays in bootloader mode and does not recognize the
// MFR_MODEL command and will return all 0xFFs for this command.
Psu[psuIndex] = new EmersonPsu((byte)(psuIndex + 1), false);
Tracer.WriteWarning("Forcing PSU instantiation to Emerson non-LES PSU at slot-{0}, Model Number: {1}",
psuIndex + 1, psuModelNumber);
}
else
{
Psu[psuIndex] = new PsuBase((byte)(psuIndex + 1));
Tracer.WriteInfo("Unidentified PSU at slot-{0}, Model Number: {1}. Default to base PSU class.", psuIndex + 1, psuModelNumber);
}
break;
}
// Enable/disable Battery Extended Operation Mode
Psu[psuIndex].SetBatteryExtendedOperationMode(ConfigLoaded.DatasafeOperationsEnabled ? true: false);
}
}
/// <summary>
/// Identifies the PSU vendor at each psu slot using the modelnumber API of the PsuBase class
/// (Assumes all PSU vendors implement the MFR_MODEL Pmbus command)
/// Based on the model number, we bind the Psu class object to the corresponding child (vendor) class object
/// </summary>
private static void PsuInitialize()
{
for (uint psuIndex = 0; psuIndex < ConfigLoaded.NumPsus; psuIndex++)
{
// Initially create instance of the base class
// Later.. based on the psu model number, we create the appropriate psu class object
Psu[psuIndex] = new PsuBase((byte)(psuIndex + 1));
PsuModelNumberPacket modelNumberPacket = new PsuModelNumberPacket();
modelNumberPacket = Psu[psuIndex].GetPsuModel();
string psuModelNumber = modelNumberPacket.ModelNumber;
PsuModel model = ConvertPsuModelNumberToPsuModel(psuModelNumber);
switch (model)
{
case PsuModel.Delta:
Psu[psuIndex] = new DeltaPsu((byte)(psuIndex + 1));
Tracer.WriteInfo("Delta Psu identified at slot-{0}, Model Number: {1}", psuIndex + 1, psuModelNumber);
break;
case PsuModel.Emerson:
Psu[psuIndex] = new EmersonPsu((byte)(psuIndex + 1));
Tracer.WriteInfo("Emerson Psu identified at slot-{0}, Model Number: {1}", psuIndex + 1, psuModelNumber);
break;
default:
// Default to Emerson PSU to enable FW update methods in the EmersonPsu class to be called.
// This is useful when previous FW update did not complete successfully and the PSU
// is not returning valid MFR_MODEL
Psu[psuIndex] = new EmersonPsu((byte)(psuIndex + 1));
Tracer.WriteInfo("Unidentified PSU at slot-{0}, Model Number: {1}. Default to Emerson PSU.", psuIndex + 1, psuModelNumber);
break;
}
}
}
/// <summary>
/// Attempt to resolve Psu Faults
/// </summary>
private static Dictionary <byte, PsuAlertFaultStatus> PsuAlertRemediate(Dictionary <byte, PsuAlertFaultType> psuFailures)
{
Dictionary <byte, PsuAlertFaultStatus> failedPsu = new Dictionary <byte, PsuAlertFaultStatus>();
foreach (KeyValuePair <byte, PsuAlertFaultType> psu in psuFailures)
{
// If firmware update is in progress, skip this PSU
if (ChassisState.PsuFwUpdateInProgress[psu.Key - 1])
{
continue;
}
lock (ChassisState.psuLock[psu.Key - 1])
{
// Log PSU faults
ChassisState.Psu[psu.Key - 1].LogPsuFaultStatus();
// Clear PSU faults, which will clear PSU_ALERT
CompletionCode clearAlert = ClearPsuFault(psu.Key);
if (clearAlert != CompletionCode.Success)
{
// PSU clear faults failed. Log failure and continue to next PSU.
failedPsu.Add(psu.Key, PsuAlertFaultStatus.PsuClearFaultFailed);
Tracer.WriteError("PsuAlertRemediate: ClearPsuFault failed on PsuId: {0}", psu.Key);
continue;
}
if (psu.Value == PsuAlertFaultType.PsuFailure)
{
// Check that the PSU is on
PsuStatusPacket psuStatus = ChassisState.Psu[psu.Key - 1].GetPsuStatus();
if (psuStatus.CompletionCode != CompletionCode.Success)
{
failedPsu.Add(psu.Key, PsuAlertFaultStatus.PsuFault);
Tracer.WriteError("PsuAlertRemediate: GetPsuStatus on PSU ({0}) failed with return code {1}",
psu.Key, psuStatus.CompletionCode);
}
else
{
if (psuStatus.PsuStatus == (byte)Contracts.PowerState.ON)
{
// Check PSU power output
PsuPowerPacket power = ChassisState.Psu[psu.Key - 1].GetPsuPower();
if ((power.CompletionCode == CompletionCode.Success) && (power.PsuPower != 0))
{
Tracer.WriteInfo("PsuStatus clear faults succeeded. Psu: {0} drawing power: {1} Watts",
psu.Key, power.PsuPower);
}
else
{
// PSU is not outputting power.
failedPsu.Add(psu.Key, PsuAlertFaultStatus.PsuNoOutputPower);
Tracer.WriteError("PsuAlertRemediate failed Psu. PsuId: {0} Psu Error State: {1}",
psu.Key, PsuAlertFaultStatus.PsuNoOutputPower.ToString());
}
}
else
{
// PSU is turned off.
failedPsu.Add(psu.Key, PsuAlertFaultStatus.PsuPowerOff);
Tracer.WriteError("PsuAlertRemediate failed Psu. PsuId: {0} Psu Error State: {1}",
psu.Key, PsuAlertFaultStatus.PsuPowerOff.ToString());
}
}
}
else if ((ConfigLoaded.BatteryMonitoringEnabled) && (ChassisState.Psu[(psu.Key - 1)] is EmersonPsu))
{
// convert psu from base class object
EmersonPsu emersonPsu = (EmersonPsu)ChassisState.Psu[(psu.Key - 1)];
if (psu.Value == PsuAlertFaultType.BatteryFault)
{
// clear battery fault status
CompletionCode clearFault = emersonPsu.ClearBatteryFaultIndicator();
if (clearFault == CompletionCode.Success)
{
EmersonPsu.BatteryFaultIndicatorPacket faultIndicator = emersonPsu.GetBatteryFaultIndicator();
if (faultIndicator.BatteryFault == 1)
{
if (!failedPsu.ContainsKey(emersonPsu.PsuId))
{
// Psu Clear faults did not succeed.
failedPsu.Add(psu.Key, PsuAlertFaultStatus.BatteryFault);
}
Tracer.WriteError("PsuAlertRemediate failed to clear battery fault. PsuId: {0} Battery Error State: {1}",
psu.Key, PsuAlertFaultStatus.BatteryFault.ToString());
}
}
}
else if (psu.Value == PsuAlertFaultType.OnBattery && ConfigLoaded.NumBatteries > 0)
{
// Check if we need to trigger delegate to process battery status
if (ConfigLoaded.ProcessBatteryStatus)
{
double sumBatteryChargeLevel = 0;
ChassisEnergyStorageStatus status = null;
// list to store battery charge levels
List <string> batteryStates = new List <string>();
// battery present or not, set to true if even one battery is present.
// default to false
bool isBatteryPresent = false;
// Calculate average battery charge level
for (int index = 1; index <= ConfigLoaded.NumBatteries; index++)
{
status = ChassisState.GetEnergyStorageStatus((byte)index);
// Add to the list battery charge levels
batteryStates.Add(status.State.ToString());
// If even one battery is present, set flag to true
if (status.Present)
{
isBatteryPresent = true;
}
// If battery state is not unknown, add up the charge level.
if (status.State != EnergyStorageState.Unknown)
{
sumBatteryChargeLevel += status.PercentCharge;
}
}
double avgChargeLevel = (sumBatteryChargeLevel / ConfigLoaded.NumBatteries);
// Process battery status if battery discharge time is greater than the allowed discharge time
// from app.config( default 35 seconds) or Average battery charge level is below a given threshold value.
if (BatteryDischargeTimer.Elapsed > new System.TimeSpan(0, 0, ConfigLoaded.BatteryDischargeTimeInSecs) ||
avgChargeLevel < ConfigLoaded.BatteryChargeLevelThreshold)
{
// Invoke method to trigger NVDIMM backup for critical battery status
ThreadPool.QueueUserWorkItem(new WaitCallback(ChassisManagerInternal.ProcessCriticalBatteryStatus));
}
// Calculate backup energy available per blade and per NVDIMM
double bladeEnergy = (ConfigLoaded.NumPsus * BATT_POUT_MAX * BATT_OP_TIME_100_LOAD * avgChargeLevel) /
ConfigLoaded.Population;
double nvdimmEnergy = (ConfigLoaded.NumPsus * BATT_POUT_EXTENDED * BATT_OP_TIME_75W_LOAD) /
(ConfigLoaded.Population * ConfigLoaded.NvDimmPerBlade);
// Scale the values
bladeEnergy = bladeEnergy / ENERGY_STORAGE_SCALING_JOULES;
nvdimmEnergy = nvdimmEnergy / ENERGY_STORAGE_SCALING_JOULES;
// Send battery status to BMC, check returned completion code for success
Dictionary <byte, CompletionCode> results = WcsBladeFacade.BroadcastSetEnergyStorage
(isBatteryPresent, GetBatteryStateToBroadcast(batteryStates), ENERGY_STORAGE_SCALING_JOULES, (ushort)bladeEnergy, (byte)nvdimmEnergy);
// Check if broadcast failed for any blade, if yes log error.
for (int index = 1; index <= ConfigLoaded.Population; index++)
{
CompletionCode code;
if (results.TryGetValue((byte)index, out code))
{
// If completion code returned is not success
if (code != CompletionCode.Success)
{
Tracer.WriteError("PsuMonitor: ProcessBatteryStatus: " +
"Failed to update battery status to BMC for blade: " + index +
", completion code returned: " + code);
}
}
else
{
// If blade entry does not exist.
Tracer.WriteError("PsuMonitor: ProcessBatteryStatus : " +
"Failed to update battery status to BMC for blade: " + index);
}
}
}
}
}
} // lock...
} // foreach...
return(failedPsu);
}
/// <summary>
/// Checks for faults on each PSU
/// </summary>
private static Dictionary <byte, PsuAlertFaultType> PsuAlertInvestigate()
{
Dictionary <byte, PsuAlertFaultType> failures = new Dictionary <byte, PsuAlertFaultType>();
// Check status for all PSU
foreach (PsuBase psu in ChassisState.Psu)
{
// If firmware update is in progress, skip this PSU
if (ChassisState.PsuFwUpdateInProgress[psu.PsuId - 1])
{
continue;
}
lock (ChassisState.psuLock[psu.PsuId - 1])
{
PsuStatusPacket psuStatus = psu.GetPsuStatus();
if (psuStatus.CompletionCode != CompletionCode.Success)
{
Tracer.WriteError("PsuAlertInvestigate: GetPsuStatus on PSU ({0}) failed with return code {1}",
psu.PsuId, psuStatus.CompletionCode);
failures.Add(psu.PsuId, PsuAlertFaultType.PsuFailure);
}
else
{
if (psuStatus.PsuStatus != (byte)Contracts.PowerState.ON)
{
// PSU is completely turned off
failures.Add(psu.PsuId, PsuAlertFaultType.PsuFailure);
}
else if ((ConfigLoaded.BatteryMonitoringEnabled) && (ChassisState.Psu[psu.PsuId - 1] is EmersonPsu))
{
// Check battery status for Emerson PSU
EmersonPsu emersonPsu = (EmersonPsu)psu;
// Get battery status and health
BatteryStatusPacket battStatus = emersonPsu.GetBatteryStatus();
EmersonPsu.BatteryHealthStatusPacket battHealth = emersonPsu.GetBatteryHealthStatus();
if ((battStatus.CompletionCode == CompletionCode.Success) &&
(battHealth.CompletionCode == CompletionCode.Success))
{
// Update chassis energy storage variables
bool batteryPresent = (battStatus.Presence == 1) ? true : false;
bool batteryFault = (battStatus.FaultDetected == 1) ? true : false;
EnergyStorageState battState = EnergyStorageState.Unknown;
if (batteryPresent)
{
if (batteryFault)
{
// Battery Fault Detected.
failures.Add(emersonPsu.PsuId, PsuAlertFaultType.BatteryFault);
Tracer.WriteError("PsuAlertInvestigate Battery Fault Detected. PsuId {0}", emersonPsu.PsuId);
}
else // if no fault detected, check if system is on battery.
{
// Determine battery state
if (battHealth.Discharging == 0)
{
battState = EnergyStorageState.Charging;
// We are charging, reset the timer.
if (BatteryDischargeTimer.IsRunning)
{
BatteryDischargeTimer.Reset();
}
}
else
{
// Emerson stated that we can have discharging even when on AC since the charger will have hysteresis.
// Hence we need to check Discharging and Battery Power Output to determine if we are on Battery.
if (battStatus.BatteryPowerOutput != 0)
{
battState = EnergyStorageState.Discharging;
}
else
{
battState = EnergyStorageState.Floating;
}
}
if (battState == EnergyStorageState.Discharging)
{
// Start the timer if not already running.
if (!BatteryDischargeTimer.IsRunning)
{
BatteryDischargeTimer.Start();
}
// Psu Battery is Discharging. System is on battery.
// Log it as a failure for processing in PsuAlertRemediate() which is called outside this method
failures.Add(emersonPsu.PsuId, PsuAlertFaultType.OnBattery);
Tracer.WriteInfo("PsuAlertInvestigate Psu Battery discharging. PsuId {0}", emersonPsu.PsuId);
}
}
}
else
{
Tracer.WriteInfo("PsuAlertInvestigate, no battery present for Psu: {0}", emersonPsu.PsuId);
}
// Update chassis energy storage values
ChassisEnergyStorageStatus chassisEnergyStatus =
new ChassisEnergyStorageStatus(batteryPresent, battState,
battStatus.BatteryChargeLevel, battStatus.BatteryPowerOutput, batteryFault);
if (!ChassisState.SetEnergyStorageStatus(emersonPsu.PsuId, chassisEnergyStatus))
{
Tracer.WriteError(
string.Format("PsuAlertInvestigate: SetEnergyStorageStatus failed for BatteryId {0}", emersonPsu.PsuId));
}
}
else
{
// Failed to get battery status or health. Log as battery fault
failures.Add(emersonPsu.PsuId, PsuAlertFaultType.BatteryFault);
Tracer.WriteError("PsuAlertInvestigate failed to get Battery Status. PsuId {0} Status Completion Code: {1} Health Completion Code: {2}",
emersonPsu.PsuId, battStatus.CompletionCode, battHealth.CompletionCode);
}
}
// If PSU is on and there are no battery faults, check if other faults are present
// Add PSU to failure list so that we can log it in PsuAlertRemediate()
if ((!failures.ContainsKey(psu.PsuId)) && (psuStatus.FaultPresent))
{
failures.Add(psu.PsuId, PsuAlertFaultType.PsuFaultPresent);
}
}
} // lock...
} // foreach...
return(failures);
}
