/// <summary>
/// Send Receive chassis messages
/// </summary>
public ChassisResponse SendReceive(DeviceType deviceType, byte deviceId, ChassisRequest chassisRequest, Type responseType, byte priority = (byte)PriorityLevel.User)
{
// Serialize the OMC request into bytes.
byte[] chassisRequestMessage = chassisRequest.GetBytes();
byte[] chassisResponseMessage;
CommunicationDevice.SendReceive((PriorityLevel)priority, (byte)deviceType, deviceId, chassisRequestMessage, out chassisResponseMessage);
// Create the response based on the provided type and message bytes.
ConstructorInfo constructorInfo = responseType.GetConstructor(Type.EmptyTypes);
ChassisResponse chassisResponse = (ChassisResponse)constructorInfo.Invoke(new Object[0]);
// Expected Packet Format:
// 4 5-6 N
// |Completion Code|Byte Count|Payload|
// 0 byte 2 byte 3+ byte
if (chassisResponseMessage.Length >= 3)
{
chassisResponse.Initialize(chassisResponseMessage, chassisResponseMessage.Length);
}
else
{
chassisResponse.CompletionCode = (byte)CompletionCode.ResponseNotProvided;
}
// Response to the OMC request message.
return(chassisResponse);
}
internal static Contracts.StartSerialResponse StartBladeSerialSession(int bladeId, int timeoutInSecs)
{
Contracts.StartSerialResponse response = new Contracts.StartSerialResponse();
response.completionCode = Contracts.CompletionCode.Failure;
response.serialSessionToken = null;
Tracer.WriteInfo("BladeSerialSessionMetadata StartBladeSerialSession({0})", bladeId);
// If there is an already existing Blade serial session (indicated by a valid bladeId and a valid sessionToken), return failure with appropriate completion code
if (CompareAndSwapMetadata(ConfigLoaded.InactiveBladePortId, ConfigLoaded.InactiveBladeSerialSessionToken, ConfigLoaded.InactiveBladePortId, ConfigLoaded.InactiveBladeSerialSessionToken) != CompletionCode.Success)
{
Tracer.WriteError("StartBladeSerialSession({0}): Start failed because of already active session.", bladeId);
response.completionCode = Contracts.CompletionCode.SerialSessionActive;
return(response);
}
// Ipmi command to indicate start of serial session
// This has to be executed before Enabling comm. dev. safe mode otherwise this will fail
Ipmi.SerialMuxSwitch sms = WcsBladeFacade.SetSerialMuxSwitch((byte)bladeId, Ipmi.MuxSwtich.SwitchSystem);
// If set serial mux fails - reverse all operations
if (sms.CompletionCode != (byte)CompletionCode.Success)
{
Tracer.WriteError("BladeSerialSessionMetadata.StartBladeSerialSession({0}): Ipmi SetSerialMuxSwitch Failed", bladeId);
if (!CommunicationDevice.DisableSafeMode())
{
Tracer.WriteError("BladeSerialSessionMetadata.StartBladeSerialSession({0}): Unable to disable comm.dev. safe mode", bladeId);
}
// Whenever we disable safe mode, make sure that no more serial session activity may be performed - by reseting metadata
if (!BladeSerialSessionMetadata.ResetMetadata())
{
Tracer.WriteError("BladeSerialSessionMetadata.StopBladeSerialSession({0}): Unable to reset metadata", bladeId);
}
return(response);
}
byte[] randomNumber = new byte[8];
new System.Security.Cryptography.RNGCryptoServiceProvider().GetNonZeroBytes(randomNumber);
// Initialize Blade Serial Session MetaData - Init function does this automically
// This function acts as a serialization point - only one active thread can proceed beyond this
if (CompareAndSwapMetadata(ConfigLoaded.InactiveBladePortId, ConfigLoaded.InactiveBladeSerialSessionToken, bladeId, BitConverter.ToString(randomNumber), DateTime.Now) != CompletionCode.Success)
{
response.completionCode = Contracts.CompletionCode.SerialSessionActive;
return(response);
}
response.completionCode = Contracts.CompletionCode.Success;
response.serialSessionToken = BitConverter.ToString(randomNumber);
// Initializing TimeoutBladeSerialSessionInSecs with user defined session timeout
ConfigLoaded.TimeoutBladeSerialSessionInSecs = timeoutInSecs;
return(response);
}
/// <summary>
/// Release Resources.
/// </summary>
public static void Dispose()
{
lock (locker)
{
// stop internal Chassis threads
ChassisManagerInternal.Halt();
// release communication layer resources.
CommunicationDevice.Release();
initialized = false;
}
}
/// <summary>
/// Initialize Communication Device
/// </summary>
/// <returns>status byte which indicates whether initialization was successful or not</returns>
private static byte CommunicationDeviceInitialize()
{
byte status = (byte)CompletionCode.UnspecifiedError;
Tracer.WriteInfo("Initializing Communication Device");
CompletionCode completionCode = CommunicationDevice.Init();
#region Comm. Device Initialization Retry
if (CompletionCodeChecker.Failed(completionCode))
{
Tracer.WriteWarning("Initialization failed: {0}", completionCode);
int loop = 0;
// Retry 3 times before failing completely
for (loop = 0; loop < ConfigLoaded.MaxRetries; loop++)
{
Tracer.WriteInfo("Initialization Retry: {0}", loop);
completionCode = CommunicationDevice.Init();
if (CompletionCodeChecker.Succeeded(completionCode))
{
break;
}
}
if (loop == ConfigLoaded.MaxRetries)
{
Tracer.WriteError("Re-attempt at Communication Device Initialization failed with code: {0}", completionCode);
return(status);
}
}
#endregion
if (CompletionCodeChecker.Succeeded(completionCode))
{
Tracer.WriteInfo("Communication Device Initialization successful..");
}
return((byte)CompletionCode.Success);
}
protected override void OnStop()
{
if (serviceHost != null)
{
serviceHost.Close();
serviceHost = null;
}
Tracer.WriteInfo("OnStop: Service closed");
RequestAdditionalTime(60 * 1000); // This is to prevent Windows service from timeouts
// Release Chassis Manager threads
this.Release();
Tracer.WriteInfo("OnStop: Chassis Manager threads stopped");
// Try to gracefully Close Open Ipmi sessions
WcsBladeFacade.Release();
Tracer.WriteInfo("OnStop: WcsBladeFacade released");
// Release the communication device layer holds
CommunicationDevice.Release();
Tracer.WriteInfo("OnStop: Communication Device released");
}
internal static Contracts.ChassisResponse StopBladeSerialSession(int bladeId, string sessionToken, bool forceKill = false)
{
Contracts.ChassisResponse response = new Contracts.ChassisResponse();
response.completionCode = Contracts.CompletionCode.Failure;
Tracer.WriteInfo("BladeSerialSessionMetadata.Received Stopbladeserialsession({0})", bladeId);
// If there is NOT an already existing Blade serial session (indicated by a invalid bladeId and a invalid sessionToken), return failure with appropriate completion code
if (CompareAndSwapMetadata(ConfigLoaded.InactiveBladePortId, ConfigLoaded.InactiveBladeSerialSessionToken,
ConfigLoaded.InactiveBladePortId, ConfigLoaded.InactiveBladeSerialSessionToken) == CompletionCode.Success)
{
Tracer.WriteError("StopBladeSerialSession({0}): Stop failed because of no active session.", bladeId);
response.completionCode = Contracts.CompletionCode.NoActiveSerialSession;
return(response);
}
// Normal scenario when forcekill option is not true.. check for bladeid correctness and if it currently holds the serial session
if (!forceKill)
{
if (ChassisManagerUtil.CheckBladeId((byte)bladeId) != (byte)CompletionCode.Success)
{
response.completionCode = Contracts.CompletionCode.ParameterOutOfRange;
return(response);
}
// If this bladeid do not currently hold the serial session, return failure
if (CompareAndSwapMetadata(bladeId, sessionToken, bladeId, sessionToken) != CompletionCode.Success)
{
response.completionCode = Contracts.CompletionCode.SerialSessionActive;
return(response);
}
}
// Communication device has to come out of safe mode - should allow IPMI commands to go to the BMC
if (!CommunicationDevice.DisableSafeMode())
{
Tracer.WriteError(
"BladeSerialSessionMetadata.StopBladeSerialSession({0}): CommunicationDevice.DisableSafeMode Failed",
bladeId);
}
Ipmi.SerialMuxSwitch rms;
// If forcekill parameter is false, then use the bladeid that is passed by the user
if (!forceKill)
{
rms = WcsBladeFacade.ResetSerialMux((byte)bladeId);
}
// If forcekill parameter is true, then use the bladeid that currently holds the serial session
else
{
rms = WcsBladeFacade.ResetSerialMux((byte)BladeSerialSessionMetadata.bladeId);
}
if (rms.CompletionCode != (byte)CompletionCode.Success)
{
Tracer.WriteError("BladeSerialSessionMetadata.StopBladeSerialSession({0}): Ipmi ReSetSerialMuxSwitch Failed", bladeId);
}
if (!BladeSerialSessionMetadata.ResetMetadata())
{
Tracer.WriteError("BladeSerialSessionMetadata.StopBladeSerialSession: Unable to reset metadata");
}
response.completionCode = Contracts.CompletionCode.Success;
// Resetting TimeoutBladeSerialSessionInSecs to 0 to account for default or user provided session timeout value
ConfigLoaded.TimeoutBladeSerialSessionInSecs = 0;
return(response);
}
/// <summary>
/// Sets the chassis fan speed
/// </summary>
private void SetAllFanSpeeds()
{
// rate limiter for setting thread
Thread.Sleep(TimeSpan.FromMilliseconds(SetTimePeriod));
// Get max requirement from the bladerequirement table
byte maxFanRequest = this.GetMaxRequirement();
if (CommunicationDevice.IsSafeMode())
{
// Set fan speed to maximum
Tracer.WriteInfo("Safe mode: Setting Fan speed to max");
maxFanRequest = MaxPWM;
}
Tracer.WriteInfo("Max value got from Blade table = {0} (at {1})", maxFanRequest, DateTime.Now);
// Check Fan Status and get number of working fans
int numFansWorking = GetAllFanStatus();
// Handle one fan failure
if (numFansWorking == MaxFanCount - 1)
{
// Alert that one fan has failed!
Tracer.WriteError("Fan failure, applying conversion");
// Denote fan failure in chassis
ChassisState.FanFailure = true;
double conversion = (double)MaxFanCount / (double)(MaxFanCount - 1);
maxFanRequest = (byte)(conversion * maxFanRequest);
}
else if (numFansWorking < MaxFanCount - 1)
{
// Set fan speed to max for fan failures more than N-1
maxFanRequest = MaxPWM; // this is to set at max speed
Tracer.WriteError("More than 1 Fans failed");
// Denote that this is a fan failure in chassis
ChassisState.FanFailure = true;
}
else
{
// All fans are working fine - check rear attention LED and if on, turn it off (by setting fanFailure to false)
ChassisState.FanFailure = false;
}
// Do altitude correction
maxFanRequest = (byte)((1 + ConfigLoaded.AltitudeCorrectionFactor * (int)(ConfigLoaded.Altitude / 1000)) * maxFanRequest);
// Bound fan request to the maximum possible
if (maxFanRequest > MaxPWM)
{
maxFanRequest = MaxPWM;
}
// Enable Ramp Down in smaller steps
if (PrevFanPWM >= maxFanRequest + 2 * ConfigLoaded.StepPWM)
{
maxFanRequest = (byte)(PrevFanPWM - ConfigLoaded.StepPWM);
}
// Set fan speed for all fans - setting one fan device is enough, since setfanspeed is for all fan devices
byte status = ChassisState.Fans[0].SetFanSpeed(maxFanRequest);
// Trace the speed of fan
Tracer.WriteInfo("Fan speed = " + ChassisState.Fans[0].GetFanSpeed());
if (status != (byte)CompletionCode.Success)
{
Tracer.WriteWarning("SetFanSpeed failed with Completion Code: {0:X}", status);
}
else
{
Tracer.WriteInfo("Fan Speed set to {0}", maxFanRequest);
}
// Store current fan PWM in PrevFanPWM for next iteration
PrevFanPWM = maxFanRequest;
}
/// <summary>
/// Function that gets all the fan speed requirements
/// from the Blade. It also updates the balde state
/// </summary>
private void GetAllBladePwmRequirements()
{
// Rate is required to timestep over each individual Blade call
double rate = (double)GetTimePeriod / (double)MaxSledCount;
double timeDiff = 0;
for (byte blade = 1; blade <= MaxSledCount; blade++)
{
// Handle shutdown state
if (ChassisState.ShutDown)
{
return;
}
// default PWM setting
byte PWM = (byte)ConfigLoaded.MinPWM;
// Query blade type from IPMI layer
ChassisState.BladeTypeCache[blade - 1] = (byte)WcsBladeFacade.clients[blade].BladeClassification;
// wait for rate limiter which includes the previous time difference for sensor get, and then issue get fan requirement
double sleepTime = rate - timeDiff;
if (sleepTime > rate)
{
sleepTime = rate;
}
if (sleepTime > 0)
{
Thread.Sleep(TimeSpan.FromMilliseconds(sleepTime));
}
if (CommunicationDevice.IsSafeMode())
{
// Do not perform any sensor reading - continue in the for loop
Tracer.WriteInfo("Monitoring thread: Safe Mode, Skipping sensor read");
continue;
}
Tracer.WriteInfo("GetBladeRequirement called at {0} for sledId {1} (state: {2})", DateTime.Now, blade,
ChassisState.GetStateName(blade));
// Check for the condition where known state is hardpoweroff, but someone plugged a new blade in
if (ChassisState.GetBladeState(blade) == (byte)BladeState.HardPowerOff)
{
ChassisState.PowerFailCount[blade - 1]++;
// TODO: identify if this period is sufficient to do this check
if (ChassisState.PowerFailCount[blade - 1] > (ConfigLoaded.MaxRetries * ConfigLoaded.Population))
{
CheckPowerEnableState(blade);
ChassisState.PowerFailCount[blade - 1] = 0;
}
}
// Log Start time
DateTime startTime = DateTime.Now;
// If blade was in Fail state
if (ChassisState.GetBladeState(blade) == (byte)BladeState.Fail)
{
// If failed count is greater than a maximum value, we move it to Initialization state
if (ChassisState.FailCount[blade - 1] > ConfigLoaded.MaxFailCount)
{
// Move to Initialization state so that this sled could be reinitialized
Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Initialization", blade,
ChassisState.GetStateName(blade));
ChassisState.SetBladeState(blade, (byte)BladeState.Initialization);
}
else
{
// Moving out of Fail state - First we use a light-weight get GUID to check whether the blade is there
DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade);
if (guid.CompletionCode == (byte)CompletionCode.Success)
{
Tracer.WriteInfo("GUID present for sled {0}, GUID: {1}", blade, guid.Guid.ToString());
// Change state to Probation
Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", blade,
ChassisState.GetStateName(blade));
ChassisState.SetBladeState(blade, (byte)BladeState.Probation);
}
else
{
Tracer.WriteInfo("Get System GUID returns a bad completion status: {0}", guid.CompletionCode);
}
}
// Increase time spent in Fail state everytime we are in this state
ChassisState.FailCount[blade - 1]++;
}
// Handles Initialization
if (ChassisState.GetBladeState(blade) == (byte)BladeState.Initialization)
{
this.ReInitialize(blade);
}
// Normal operation - possible states are probation or healthy
if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation ||
ChassisState.GetBladeState(blade) == (byte)BladeState.Healthy)
{
if (ChassisState.GetBladeType(blade) == (byte)BladeType.Jbod)
{
DeviceGuid guid = WcsBladeFacade.GetSystemGuid(blade);
if (guid.CompletionCode == (byte)CompletionCode.Success)
{
Tracer.WriteInfo("GUID present for jbod {0}, GUID: {1}", blade, guid.Guid.ToString());
// Change state to Probation
Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Healthy", blade, ChassisState.GetStateName(blade));
ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
}
else
{
Tracer.WriteInfo("Get System GUID for jbod {0} failed with status {1}", blade, guid.CompletionCode);
// Set it to failed state, where we will retry guids and reinitialize if needed
Tracer.WriteInfo("State Transition for jbod {0}: {1} -> Fail", blade, ChassisState.GetStateName(blade));
ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
}
// No need to check for sensor reading, just continue
continue;
}
// Call temperature reading list command
SensorReading Temps = WcsBladeFacade.GetSensorReading((byte)blade, (byte)ConfigLoaded.InputSensor, PriorityLevel.System);
if (Temps.CompletionCode != (byte)CompletionCode.Success)
{
Tracer.WriteWarning("SledId: {0} - getTempSensorReading failed with code {1:X}", blade, Temps.CompletionCode);
// Move to Fail state if no readings were obtained
Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Fail", blade,
ChassisState.GetStateName(blade));
ChassisState.SetBladeState(blade, (byte)BladeState.Fail);
}
else
{
Tracer.WriteInfo("#### Sledid= " + blade + " Sensor id= " + ConfigLoaded.InputSensor + " Sensor reading= " +
Temps.Reading + " Raw= " + Temps.RawReading + ", LowerNonCritical= " +
ConfigLoaded.SensorLowThreshold + ", UpperNonCritical= " + ConfigLoaded.SensorHighThreshold);
// Handle state logic if needed
// Probation state should be shifted to Healthy since there was no timeout, & sensorread succeeded
if (ChassisState.GetBladeState(blade) == (byte)BladeState.Probation)
{
// Change state to healthy
Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Healthy", blade,
ChassisState.GetStateName(blade));
ChassisState.SetBladeState(blade, (byte)BladeState.Healthy);
ChassisState.FailCount[blade - 1] = 0; // reset the fail count
}
PWM = GetPwmFromTemperature(Temps.Reading,
ConfigLoaded.SensorLowThreshold,
ConfigLoaded.SensorHighThreshold);
Tracer.WriteInfo("PWM value for Sensor {0} = {1}", InputSensor, PWM);
}
}
// write value into requirements table
BladeRequirementTable[blade - 1] = PWM;
// Log end time and capture time of execution for sensor get command
DateTime endTime = DateTime.Now;
timeDiff = endTime.Subtract(startTime).TotalMilliseconds; // convert time difference into milliseconds
}
}
/// <summary>
/// Initialize Chassis constants and configs
/// </summary>
internal byte Initialize()
{
Tracer.WriteInfo("Initializing state");
byte status = (byte)CompletionCode.UnspecifiedError;
ChassisState.Initialize();
Tracer.WriteInfo("Initializing Communication Device");
// Initializer lower layer communication device
CompletionCode completionCode = CommunicationDevice.Init();
if (CompletionCodeChecker.Failed(completionCode))
{
Tracer.WriteWarning("Initialization failed: {0}", completionCode);
int loop = 0;
// Retry 3 times before failing completely
for (loop = 0; loop < ConfigLoaded.MaxRetries; loop++)
{
Tracer.WriteInfo("Initialization Retry: {0}", loop);
completionCode = CommunicationDevice.Init();
if (CompletionCodeChecker.Succeeded(completionCode))
{
break;
}
}
if (loop == ConfigLoaded.MaxRetries)
{
Tracer.WriteError("Re-attempt at Communication Device Initialization failed with code: {0}", completionCode);
return(status);
}
}
if (CompletionCodeChecker.Succeeded(completionCode))
{
Tracer.WriteInfo("Communication Device Initialized");
status = (byte)CompletionCode.Success;
}
// Get power status of enable pin for each blade and update blade state
for (byte deviceId = 1; deviceId <= MaxSledCount; deviceId++)
{
CheckPowerEnableState(deviceId);
}
// Initialize Wcs Blade - TODO: This initialize should return some status
WcsBladeFacade.Initialize(); // This method just creates IPMI Client Class for each blade.
Tracer.WriteInfo("IPMI Facade Initialized, Number of blades initialized: {0}", WcsBladeFacade.Initialized);
// check all client initialization status and update state
Tracer.WriteInfo("Checking client status for {0} blades", MaxSledCount);
for (byte deviceId = 1; deviceId <= MaxSledCount; deviceId++)
{
// TODO: How to check initialized status, now that this has become a function
if (WcsBladeFacade.clients[deviceId].Initialize()) // This method logs on to an IPMI session.
{
// If initialized is true, change state to probation
Tracer.WriteInfo("State Transition for Sled {0}: {1} -> Probation", deviceId,
ChassisState.GetStateName(deviceId));
ChassisState.SetBladeState(deviceId, (byte)BladeState.Probation);
}
else
{
Tracer.WriteInfo("Blade not initialized: Blade ", +deviceId);
}
}
Tracer.WriteInfo("Initializing Watchdog Timer");
// Initialize WatchDog Timer
ChassisState.Wdt.EnableWatchDogTimer();
Tracer.WriteInfo("Watchdog timer initialized");
// Initialize internal chassis manager tables
this.ChassisInternalInitialize();
return(status);
}