/// <summary>
/// Single point entry method for all datasafe command execution and ADR triggers via CM (including battery panic)
/// Per-blade actions are executed atomically
/// </summary>
/// <param name="bladeId">target bladeId</param>
/// <param name="command">Command that triggered ADR</param>
/// <param name="command">Use BatteryLowCapacityPanic when battery discharging and critical capacity</param>
/// <param name="command">Use DoNothing for default and when battery transitions from discharging to charging</param>
/// Return DatasafeCommandsReturnStatus.CommandExecuted if command successfully executed
/// Return DatasafeCommandsReturnStatus.CommandDelayed if command successfully delayed
/// Return DatasafeCommandsReturnStatus.CommandFailed if command execution failed
private static DatasafeBladeStatus ProcessDatasafeActionHelper(int bladeId, DatasafeActions command)
{
DatasafeBladeStatus response = new DatasafeBladeStatus();
response.status = DatasafeCommandsReturnStatus.CommandFailed;
// Backup status variables
NvdimmBackupStatus backupStatus = NvdimmBackupStatus.Unknown;
int nvdimmPresentTimeRemaining;
byte adrCompleteDelay;
byte nvdimmPresentPowerOffDelay;
// Get Backup status
bool getBackupStatusOK = GetBackupStatus(bladeId, out backupStatus, out nvdimmPresentTimeRemaining,
out adrCompleteDelay, out nvdimmPresentPowerOffDelay);
if (getBackupStatusOK)
{
response.isBackupPending = backupStatus == NvdimmBackupStatus.Pending ? true : false;
response.remainingBackupDuration = nvdimmPresentTimeRemaining;
}
// DoNothing command will be used to get status of datasafe backup operation
if (command == DatasafeActions.DoNothing)
{
response.status = getBackupStatusOK == true ?
DatasafeCommandsReturnStatus.CommandExecuted : DatasafeCommandsReturnStatus.CommandFailed;
return response;
}
switch (command)
{
case DatasafeActions.PowerCycle:
case DatasafeActions.PowerOff:
case DatasafeActions.BladeOff:
case DatasafeActions.BatteryLowCapacityPanic:
// Backup is NOT pending OR Not sure if backup is pending
if (backupStatus == NvdimmBackupStatus.NotPending || backupStatus == NvdimmBackupStatus.Unknown)
{
// Kill any active serial session - use the force kill option with invalid blade id since we do not know
// what blade currently has open serial session and the session token is also not known to us
Contracts.ChassisResponse killSessionResponse = BladeSerialSessionMetadata.StopBladeSerialSession(0, null, true);
if ((killSessionResponse.completionCode != Contracts.CompletionCode.NoActiveSerialSession) &&
(killSessionResponse.completionCode != Contracts.CompletionCode.Success))
{
Tracer.WriteError("Before SetNvDimmTrigger Datasafe Operation (BladeId-{0}): Cannot kill the serial session.", bladeId);
// It makes sense to proceed further as a best effort even if we cannot kill the serial session.. so continue..
}
// Send ADR trigger
if (WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, true,
adrCompleteDelay, nvdimmPresentPowerOffDelay))
{
response.status = DatasafeCommandsReturnStatus.CommandDelayed;
}
else
{
response.status = DatasafeCommandsReturnStatus.CommandFailed;
}
}
else if (backupStatus == NvdimmBackupStatus.Complete)
{
// We need to process all these above commands once the timer expires
if (ExecuteBladePowerCommand(bladeId, command))
{
response.status = DatasafeCommandsReturnStatus.CommandExecuted;
}
else
{
response.status = DatasafeCommandsReturnStatus.CommandFailed;
}
}
else if (backupStatus == NvdimmBackupStatus.Pending)
{
response.status = DatasafeCommandsReturnStatus.CommandDelayed;
}
break;
case DatasafeActions.PowerOn:
case DatasafeActions.BladeOn:
// Just execute command for PowerOn and BladeOn.
if (ExecuteBladePowerCommand(bladeId, command))
{
response.status = DatasafeCommandsReturnStatus.CommandExecuted;
}
else
{
response.status = DatasafeCommandsReturnStatus.CommandFailed;
}
break;
default:
break;
}
return response;
}
/// <summary>
/// Single point entry method for all datasafe command execution and ADR triggers via CM (including battery panic)
/// Per-blade actions are executed atomically
/// </summary>
/// <param name="bladeId">target bladeId</param>
/// <param name="command">Command that triggered ADR</param>
/// <param name="command">Use BatteryLowCapacityPanic when battery discharging and critical capacity</param>
/// <param name="command">Use DoNothing for default and when battery transitions from discharging to charging</param>
/// Return DatasafeCommandsReturnStatus.CommandExecuted if command successfully executed
/// Return DatasafeCommandsReturnStatus.CommandDelayed if command successfully delayed
/// Return DatasafeCommandsReturnStatus.CommandFailed if command execution failed
private static DatasafeBladeStatus ProcessDatasafeActionHelper(int bladeId, DatasafeActions command)
{
DatasafeBladeStatus response = new DatasafeBladeStatus();
response.status = DatasafeCommandsReturnStatus.CommandFailed;
// Backup status variables
NvdimmBackupStatus backupStatus = NvdimmBackupStatus.Unknown;
int nvdimmPresentTimeRemaining;
byte adrCompleteDelay;
byte nvdimmPresentPowerOffDelay;
// Get Backup status
bool getBackupStatusOK = GetBackupStatus(bladeId, out backupStatus, out nvdimmPresentTimeRemaining,
out adrCompleteDelay, out nvdimmPresentPowerOffDelay);
if (getBackupStatusOK)
{
response.isBackupPending = backupStatus == NvdimmBackupStatus.Pending ? true : false;
response.remainingBackupDuration = nvdimmPresentTimeRemaining;
response.status = DatasafeCommandsReturnStatus.CommandExecuted;
Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), action({1}), nvdimmPresentTimeRemaining({2})", bladeId, command.ToString(), nvdimmPresentTimeRemaining);
}
else
{
Tracer.WriteWarning("ProcessDatasafeActionHelper: GetBackupStatus failed for bladeId({0}), action({1})", bladeId, command.ToString());
response.status = DatasafeCommandsReturnStatus.CommandFailed;
}
// DoNothing command will be used to get status of datasafe backup operation
if (command == DatasafeActions.DoNothing)
{
return response;
}
switch (command)
{
case DatasafeActions.PowerCycle:
case DatasafeActions.PowerOff:
case DatasafeActions.BladeOff:
case DatasafeActions.BatteryLowCapacityPanic:
// If backup is complete (or timeout), execute the blade power command
if (backupStatus == NvdimmBackupStatus.Complete)
{
Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), Execute action({1})", bladeId, command.ToString());
response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
DatasafeCommandsReturnStatus.CommandFailed;
}
// If 'forcePowerActionTimeoutInSecs' time has elapsed since the initiation of backup, force blade power action - MAY LOSE DATA
else if (GetDatasafeAction(bladeId).timeInAction != DateTime.MaxValue && GetDatasafeAction(bladeId).timeInAction.Add(TimeSpan.FromSeconds(forcePowerActionTimeoutInSecs)) < DateTime.Now)
{
Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) datasafe operation pending for ({1})s.. Forcing nvdimm power action.",
bladeId, forcePowerActionTimeoutInSecs);
response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
DatasafeCommandsReturnStatus.CommandFailed;
}
// Backup is NOT pending OR Not sure if backup is pending (or timeout), send ADR trigger and initiate backup
else if (backupStatus == NvdimmBackupStatus.NotPending || backupStatus == NvdimmBackupStatus.Unknown)
{
Tracer.WriteInfo("ProcessDatasafeActionHelper: SetNvdimmTrigger bladeId({0}), action({1})", bladeId, command.ToString());
response.status = WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, true, adrCompleteDelay,
nvdimmPresentPowerOffDelay) ? DatasafeCommandsReturnStatus.CommandDelayed : DatasafeCommandsReturnStatus.CommandFailed;
}
// Backup has been pending for a long time -- 'forceTriggerTimeoutInSecs' -- force NVDIMM trigger
else if (backupStatus == NvdimmBackupStatus.Pending &&
(GetDatasafeAction(bladeId).timeInAction != DateTime.MaxValue && GetDatasafeAction(bladeId).timeInAction.Add(TimeSpan.FromSeconds(forceTriggerTimeoutInSecs)) < DateTime.Now))
{
Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) datasafe operation pending for ({1})s.. Forcing nvdimm trigger.",
bladeId, forceTriggerTimeoutInSecs);
response.status = WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, true, adrCompleteDelay,
nvdimmPresentPowerOffDelay) ? DatasafeCommandsReturnStatus.CommandDelayed : DatasafeCommandsReturnStatus.CommandFailed;
}
// Backup is pending, send a command delayed message back to the user
else if (backupStatus == NvdimmBackupStatus.Pending)
{
Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), delay action({1})", bladeId, command.ToString());
response.status = DatasafeCommandsReturnStatus.CommandDelayed;
}
else
{
Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) Unreachable code action({1}) backupstatus({2})",
bladeId, command.ToString(), backupStatus.ToString());
}
break;
case DatasafeActions.PowerOn:
case DatasafeActions.BladeOn:
// Backup is complete, NOT pending OR Not sure if backup is pending, execute on command
if (backupStatus == NvdimmBackupStatus.Complete || backupStatus == NvdimmBackupStatus.NotPending || backupStatus == NvdimmBackupStatus.Unknown)
{
Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), Execute action({1})", bladeId, command.ToString());
// Just execute command for PowerOn and BladeOn.
response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
DatasafeCommandsReturnStatus.CommandFailed;
}
// Backup has been pending for a long time -- 'forcePowerActionTimeoutInSecs' -- force blade power action - MAY LOSE DATA
else if (GetDatasafeAction(bladeId).timeInAction != DateTime.MaxValue &&
GetDatasafeAction(bladeId).timeInAction.Add(TimeSpan.FromSeconds(forcePowerActionTimeoutInSecs)) < DateTime.Now)
{
Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) datasafe operation pending for ({1})s.. Forcing nvdimm power action.",
bladeId, forcePowerActionTimeoutInSecs);
response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
DatasafeCommandsReturnStatus.CommandFailed;
}
// If backup is pending, send a command delayed message back to the user
else if(backupStatus==NvdimmBackupStatus.Pending)
{
Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), delay action({1})", bladeId, command.ToString());
response.status = DatasafeCommandsReturnStatus.CommandDelayed;
}
else
{
Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) Unreachable code action({1}) backupstatus({2})",
bladeId, command.ToString(), backupStatus.ToString());
}
break;
case DatasafeActions.EnableDatasafe:
WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, false,
adrCompleteDelay, nvdimmPresentPowerOffDelay);
break;
case DatasafeActions.DisableDatasafe:
WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.Disabled, false,
adrCompleteDelay, nvdimmPresentPowerOffDelay);
break;
default:
break;
}
return response;
}
/// <summary>
/// Processes the datasafe action.
/// This method will be called by each of DataSafe REST commands in ChassisManager.cs to process the Datasafe commands
/// </summary>
/// <param name="bladeId">The blade identifier.</param>
/// <param name="action">The action.</param>
/// <returns></returns>
internal static DatasafeBladeStatus ProcessDatasafeAction(int bladeId, DatasafeActions action)
{
DatasafeBladeStatus response = new DatasafeBladeStatus();
lock (bladeActionLock[bladeId - 1])
{
if (GetDatasafeAction(bladeId) == DatasafeActions.DoNothing)
{
// Process datasafe action
response = ProcessDatasafeActionHelper(bladeId, action);
if (response.status == DatasafeCommandsReturnStatus.CommandDelayed)
{
// Set datasafe action for this blade and trigger the delegate to process the pending datasafe command
SetDatasafeAction(bladeId, action);
pendingDatasafeEvent.Set();
}
}
else
{
// There is already an existing action for this blade. Do not accept new commands. Return failure
response.status = DatasafeCommandsReturnStatus.CommandFailed;
}
return response;
}
}
/// <summary>
/// Processes the datasafe action.
/// This method will be called by each of DataSafe REST commands in ChassisManager.cs to process the Datasafe commands
/// </summary>
/// <param name="bladeId">The blade identifier.</param>
/// <param name="action">The action.</param>
/// <returns></returns>
internal static DatasafeBladeStatus ProcessDatasafeAction(int bladeId, DatasafeActions action)
{
DatasafeBladeStatus response = new DatasafeBladeStatus();
lock (bladeActionLock[bladeId - 1])
{
// If no datasafe operation is pending on this blade, then proceed to process the new datasafe operation
if (GetDatasafeAction(bladeId).action == DatasafeActions.DoNothing || action==DatasafeActions.DoNothing)
{
// Process datasafe action
response = ProcessDatasafeActionHelper(bladeId, action);
// If the datasafe operation is asynchronous or pending
if (response.status == DatasafeCommandsReturnStatus.CommandDelayed)
{
// Set the new datasafe action for this blade
SetDatasafeAction(bladeId, action);
// Also, trigger the delegate to process the pending/asynchronous datasafe operation
pendingDatasafeEvent.Set();
}
else // If the datasafe operation is synchronous, then return the response packet
{
// TODO: Should we set the datasafe action to DoNothing here
// SetDatasafeAction(bladeId, DatasafeActions.DoNothing);
}
}
else // There is already an existing pending datasafe operation for this blade. Do not accept new commands. Return failure
{
response.status = DatasafeCommandsReturnStatus.CommandFailed;
}
return response;
}
}