Example #1
0
        /// <summary>
        /// Single point entry method for all datasafe command execution and ADR triggers via CM (including battery panic)
        /// Per-blade actions are executed atomically
        /// </summary>
        /// <param name="bladeId">target bladeId</param>
        /// <param name="command">Command that triggered ADR</param>
        /// <param name="command">Use BatteryLowCapacityPanic when battery discharging and critical capacity</param>
        /// <param name="command">Use DoNothing for default and when battery transitions from discharging to charging</param>
        /// Return DatasafeCommandsReturnStatus.CommandExecuted if command successfully executed 
        /// Return DatasafeCommandsReturnStatus.CommandDelayed if command successfully delayed
        /// Return DatasafeCommandsReturnStatus.CommandFailed if command execution failed
        private static DatasafeBladeStatus ProcessDatasafeActionHelper(int bladeId, DatasafeActions command)
        {
            DatasafeBladeStatus response = new DatasafeBladeStatus();
            response.status = DatasafeCommandsReturnStatus.CommandFailed;

            // Backup status variables
            NvdimmBackupStatus backupStatus = NvdimmBackupStatus.Unknown;
            int nvdimmPresentTimeRemaining;
            byte adrCompleteDelay;
            byte nvdimmPresentPowerOffDelay;

            // Get Backup status
            bool getBackupStatusOK = GetBackupStatus(bladeId, out backupStatus, out nvdimmPresentTimeRemaining,
                out adrCompleteDelay, out nvdimmPresentPowerOffDelay);
            if (getBackupStatusOK)
            {
                response.isBackupPending = backupStatus == NvdimmBackupStatus.Pending ? true : false;
                response.remainingBackupDuration = nvdimmPresentTimeRemaining;
            }

            // DoNothing command will be used to get status of datasafe backup operation
            if (command == DatasafeActions.DoNothing)
            {
                response.status = getBackupStatusOK == true ?
                    DatasafeCommandsReturnStatus.CommandExecuted : DatasafeCommandsReturnStatus.CommandFailed;
                return response;
            }

            switch (command)
            {
                case DatasafeActions.PowerCycle:
                case DatasafeActions.PowerOff:
                case DatasafeActions.BladeOff:
                case DatasafeActions.BatteryLowCapacityPanic:

                    // Backup is NOT pending OR Not sure if backup is pending
                    if (backupStatus == NvdimmBackupStatus.NotPending || backupStatus == NvdimmBackupStatus.Unknown)
                    {
                        // Kill any active serial session - use the force kill option with invalid blade id since we do not know
                        // what blade currently has open serial session and the session token is also not known to us
                        Contracts.ChassisResponse killSessionResponse = BladeSerialSessionMetadata.StopBladeSerialSession(0, null, true);
                        if ((killSessionResponse.completionCode != Contracts.CompletionCode.NoActiveSerialSession) &&
                            (killSessionResponse.completionCode != Contracts.CompletionCode.Success))
                        {
                            Tracer.WriteError("Before SetNvDimmTrigger Datasafe Operation (BladeId-{0}): Cannot kill the serial session.", bladeId);
                            // It makes sense to proceed further as a best effort even if we cannot kill the serial session.. so continue..
                        }

                        // Send ADR trigger
                        if (WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, true,
                            adrCompleteDelay, nvdimmPresentPowerOffDelay))
                        {
                            response.status = DatasafeCommandsReturnStatus.CommandDelayed;
                        }
                        else
                        {
                            response.status = DatasafeCommandsReturnStatus.CommandFailed;
                        }
                    }
                    else if (backupStatus == NvdimmBackupStatus.Complete)
                    {
                        // We need to process all these above commands once the timer expires
                        if (ExecuteBladePowerCommand(bladeId, command))
                        {
                            response.status = DatasafeCommandsReturnStatus.CommandExecuted;
                        }
                        else
                        {
                            response.status = DatasafeCommandsReturnStatus.CommandFailed;
                        }
                    }
                    else if (backupStatus == NvdimmBackupStatus.Pending)
                    {
                        response.status = DatasafeCommandsReturnStatus.CommandDelayed;
                    }

                    break;

                case DatasafeActions.PowerOn:
                case DatasafeActions.BladeOn:

                    // Just execute command for PowerOn and BladeOn.
                    if (ExecuteBladePowerCommand(bladeId, command))
                    {
                        response.status = DatasafeCommandsReturnStatus.CommandExecuted;
                    }
                    else
                    {
                        response.status = DatasafeCommandsReturnStatus.CommandFailed;
                    }
                    break;

                default:
                    break;
            }

            return response;
        }
        /// <summary>
        /// Single point entry method for all datasafe command execution and ADR triggers via CM (including battery panic)
        /// Per-blade actions are executed atomically
        /// </summary>
        /// <param name="bladeId">target bladeId</param>
        /// <param name="command">Command that triggered ADR</param>
        /// <param name="command">Use BatteryLowCapacityPanic when battery discharging and critical capacity</param>
        /// <param name="command">Use DoNothing for default and when battery transitions from discharging to charging</param>
        /// Return DatasafeCommandsReturnStatus.CommandExecuted if command successfully executed 
        /// Return DatasafeCommandsReturnStatus.CommandDelayed if command successfully delayed
        /// Return DatasafeCommandsReturnStatus.CommandFailed if command execution failed
        private static DatasafeBladeStatus ProcessDatasafeActionHelper(int bladeId, DatasafeActions command)
        {
            DatasafeBladeStatus response = new DatasafeBladeStatus();
            response.status = DatasafeCommandsReturnStatus.CommandFailed;

            // Backup status variables
            NvdimmBackupStatus backupStatus = NvdimmBackupStatus.Unknown;
            int nvdimmPresentTimeRemaining;
            byte adrCompleteDelay;
            byte nvdimmPresentPowerOffDelay;

            // Get Backup status
            bool getBackupStatusOK = GetBackupStatus(bladeId, out backupStatus, out nvdimmPresentTimeRemaining,
                out adrCompleteDelay, out nvdimmPresentPowerOffDelay);

            if (getBackupStatusOK)
            {
                response.isBackupPending = backupStatus == NvdimmBackupStatus.Pending ? true : false;
                response.remainingBackupDuration = nvdimmPresentTimeRemaining;
                response.status = DatasafeCommandsReturnStatus.CommandExecuted;
                Tracer.WriteInfo("ProcessDatasafeActionHelper:  bladeId({0}), action({1}), nvdimmPresentTimeRemaining({2})", bladeId, command.ToString(), nvdimmPresentTimeRemaining);
            }
            else
            {
                Tracer.WriteWarning("ProcessDatasafeActionHelper: GetBackupStatus failed for bladeId({0}), action({1})", bladeId, command.ToString());
                response.status = DatasafeCommandsReturnStatus.CommandFailed;
            }

            // DoNothing command will be used to get status of datasafe backup operation
            if (command == DatasafeActions.DoNothing)
            {
                return response;
            }

            switch (command)
            {
                case DatasafeActions.PowerCycle:
                case DatasafeActions.PowerOff:
                case DatasafeActions.BladeOff:
                case DatasafeActions.BatteryLowCapacityPanic:

                    // If backup is complete (or timeout), execute the blade power command
                    if (backupStatus == NvdimmBackupStatus.Complete)
                    {
                        Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), Execute action({1})", bladeId, command.ToString());
                        response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
                            DatasafeCommandsReturnStatus.CommandFailed;
                    }
                    // If 'forcePowerActionTimeoutInSecs' time has elapsed since the initiation of backup, force blade power action - MAY LOSE DATA
                    else if (GetDatasafeAction(bladeId).timeInAction != DateTime.MaxValue && GetDatasafeAction(bladeId).timeInAction.Add(TimeSpan.FromSeconds(forcePowerActionTimeoutInSecs)) < DateTime.Now)
                    {
                        Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) datasafe operation pending for ({1})s.. Forcing nvdimm power action.",
                            bladeId, forcePowerActionTimeoutInSecs);
                        response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
                           DatasafeCommandsReturnStatus.CommandFailed;
                    }
                    // Backup is NOT pending OR Not sure if backup is pending (or timeout), send ADR trigger and initiate backup
                    else if (backupStatus == NvdimmBackupStatus.NotPending || backupStatus == NvdimmBackupStatus.Unknown)
                    {
                        Tracer.WriteInfo("ProcessDatasafeActionHelper: SetNvdimmTrigger bladeId({0}), action({1})", bladeId, command.ToString());
                        response.status = WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, true, adrCompleteDelay,
                            nvdimmPresentPowerOffDelay) ? DatasafeCommandsReturnStatus.CommandDelayed : DatasafeCommandsReturnStatus.CommandFailed;
                    }
                    // Backup has been pending for a long time -- 'forceTriggerTimeoutInSecs' -- force NVDIMM trigger
                    else if (backupStatus == NvdimmBackupStatus.Pending &&
                         (GetDatasafeAction(bladeId).timeInAction != DateTime.MaxValue && GetDatasafeAction(bladeId).timeInAction.Add(TimeSpan.FromSeconds(forceTriggerTimeoutInSecs)) < DateTime.Now))
                    {
                        Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) datasafe operation pending for ({1})s.. Forcing nvdimm trigger.",
                            bladeId, forceTriggerTimeoutInSecs);
                        response.status = WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, true, adrCompleteDelay,
                            nvdimmPresentPowerOffDelay) ? DatasafeCommandsReturnStatus.CommandDelayed : DatasafeCommandsReturnStatus.CommandFailed;
                    }
                    // Backup is pending, send a command delayed message back to the user
                    else if (backupStatus == NvdimmBackupStatus.Pending)
                    {
                        Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), delay action({1})", bladeId, command.ToString());
                        response.status = DatasafeCommandsReturnStatus.CommandDelayed;
                    }
                    else
                    {
                        Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) Unreachable code action({1}) backupstatus({2})",
                            bladeId, command.ToString(), backupStatus.ToString());
                    }

                    break;

                case DatasafeActions.PowerOn:
                case DatasafeActions.BladeOn:

                    // Backup is complete, NOT pending OR Not sure if backup is pending, execute on command
                    if (backupStatus == NvdimmBackupStatus.Complete || backupStatus == NvdimmBackupStatus.NotPending || backupStatus == NvdimmBackupStatus.Unknown)
                    {
                        Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), Execute action({1})", bladeId, command.ToString());
                        // Just execute command for PowerOn and BladeOn.
                        response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
                                DatasafeCommandsReturnStatus.CommandFailed;
                    }
                    // Backup has been pending for a long time -- 'forcePowerActionTimeoutInSecs' -- force blade power action - MAY LOSE DATA
                    else if (GetDatasafeAction(bladeId).timeInAction != DateTime.MaxValue &&
                        GetDatasafeAction(bladeId).timeInAction.Add(TimeSpan.FromSeconds(forcePowerActionTimeoutInSecs)) < DateTime.Now)
                    {
                        Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) datasafe operation pending for ({1})s.. Forcing nvdimm power action.",
                            bladeId, forcePowerActionTimeoutInSecs);
                        response.status = ExecuteBladePowerCommand(bladeId, command) ? DatasafeCommandsReturnStatus.CommandExecuted :
                           DatasafeCommandsReturnStatus.CommandFailed;
                    }
                    // If backup is pending, send a command delayed message back to the user
                    else if(backupStatus==NvdimmBackupStatus.Pending)
                    {
                        Tracer.WriteInfo("ProcessDatasafeActionHelper: bladeId({0}), delay action({1})", bladeId, command.ToString());
                        response.status = DatasafeCommandsReturnStatus.CommandDelayed;
                    }
                    else
                    {
                        Tracer.WriteWarning("ProcessDatasafeActionHelper: bladeId({0}) Unreachable code action({1}) backupstatus({2})",
                            bladeId, command.ToString(), backupStatus.ToString());
                    }

                    break;

                case DatasafeActions.EnableDatasafe:

                    WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.PchAdrGpi, false,
                            adrCompleteDelay, nvdimmPresentPowerOffDelay);
                    break;

                case DatasafeActions.DisableDatasafe:

                    WcsBladeFacade.SetNvDimmTrigger((byte)bladeId, Ipmi.NvDimmTriggerAction.Disabled, false,
                            adrCompleteDelay, nvdimmPresentPowerOffDelay);
                    break;

                default:
                    break;
            }

            return response;
        }
Example #3
0
        /// <summary>
        /// Processes the datasafe action.
        /// This method will be called by each of DataSafe REST commands in ChassisManager.cs to process the Datasafe commands
        /// </summary>
        /// <param name="bladeId">The blade identifier.</param>
        /// <param name="action">The action.</param>
        /// <returns></returns>
        internal static DatasafeBladeStatus ProcessDatasafeAction(int bladeId, DatasafeActions action)
        {
            DatasafeBladeStatus response = new DatasafeBladeStatus();

            lock (bladeActionLock[bladeId - 1])
            {
                if (GetDatasafeAction(bladeId) == DatasafeActions.DoNothing)
                {
                    // Process datasafe action
                    response = ProcessDatasafeActionHelper(bladeId, action);
                    if (response.status == DatasafeCommandsReturnStatus.CommandDelayed)
                    {
                        // Set datasafe action for this blade and trigger the delegate to process the pending datasafe command
                        SetDatasafeAction(bladeId, action);
                        pendingDatasafeEvent.Set();
                    }
                }
                else
                {
                    // There is already an existing action for this blade. Do not accept new commands. Return failure
                    response.status = DatasafeCommandsReturnStatus.CommandFailed;
                }

                return response;
            }
        }
        /// <summary>
        /// Processes the datasafe action.
        /// This method will be called by each of DataSafe REST commands in ChassisManager.cs to process the Datasafe commands
        /// </summary>
        /// <param name="bladeId">The blade identifier.</param>
        /// <param name="action">The action.</param>
        /// <returns></returns>
        internal static DatasafeBladeStatus ProcessDatasafeAction(int bladeId, DatasafeActions action)
        {
            DatasafeBladeStatus response = new DatasafeBladeStatus();

            lock (bladeActionLock[bladeId - 1])
            {
                // If no datasafe operation is pending on this blade, then proceed to process the new datasafe operation
                if (GetDatasafeAction(bladeId).action == DatasafeActions.DoNothing || action==DatasafeActions.DoNothing)
                {
                    // Process datasafe action
                    response = ProcessDatasafeActionHelper(bladeId, action);

                    // If the datasafe operation is asynchronous or pending
                    if (response.status == DatasafeCommandsReturnStatus.CommandDelayed)
                    {
                        // Set the new datasafe action for this blade
                        SetDatasafeAction(bladeId, action);
                        // Also, trigger the delegate to process the pending/asynchronous datasafe operation
                        pendingDatasafeEvent.Set();
                    }
                    else // If the datasafe operation is synchronous, then return the response packet
                    {
                        // TODO: Should we set the datasafe action to DoNothing here
                        // SetDatasafeAction(bladeId, DatasafeActions.DoNothing);
                    }
                }
                else // There is already an existing pending datasafe operation for this blade. Do not accept new commands. Return failure
                {
                    response.status = DatasafeCommandsReturnStatus.CommandFailed;
                }

                return response;
            }
        }