/// <summary>
/// Handles the Client Status Returned by Log Parsing and then determines what values to feed the DetermineStatus routine.
/// </summary>
private void HandleReturnedStatus(SlotStatus returnedStatus, SlotModel slot)
{
var statusData = new LegacyClientStatusData
{
ClientName = Settings.Name,
SlotType = slot.UnitInfoModel.UnitInfoData.SlotType,
UnitRetrievalTime = slot.UnitInfoModel.UnitInfoData.UnitRetrievalTime,
UtcOffsetIsZero = Settings.UtcOffsetIsZero,
UtcOffset = TimeZone.CurrentTimeZone.GetUtcOffset(DateTime.Now),
ClientTimeOffset = Settings.ClientTimeOffset,
TimeOfLastUnitStart = slot.TimeOfLastUnitStart,
TimeOfLastFrameProgress = slot.TimeOfLastFrameProgress,
CurrentStatus = slot.Status,
ReturnedStatus = returnedStatus,
FrameTime = slot.UnitInfoModel.GetRawTime(Prefs.Get <PpdCalculationType>(Preference.PpdCalculation)),
BenchmarkAverageFrameTime = GetBenchmarkAverageFrameTimeOrDefault(slot.UnitInfo),
TimeOfLastFrame = slot.UnitInfoModel.UnitInfoData.CurrentFrame == null
? TimeSpan.Zero
: slot.UnitInfoModel.UnitInfoData.CurrentFrame.TimeOfFrame,
UnitStartTimeStamp = slot.UnitInfoModel.UnitInfoData.UnitStartTimeStamp,
AllowRunningAsync = Prefs.Get <bool>(Preference.AllowRunningAsync)
};
SlotStatus computedStatus = LegacyClientStatus.GetSlotStatus(statusData, Logger);
// If the returned status is EuePause and current status is not
if (computedStatus.Equals(SlotStatus.EuePause) && statusData.CurrentStatus.Equals(SlotStatus.EuePause) == false)
{
if (Prefs.Get <bool>(Preference.EmailReportingEnabled) &&
Prefs.Get <bool>(Preference.ReportEuePause))
{
SendEuePauseEmail(statusData.ClientName);
}
}
// If the returned status is Hung and current status is not
if (computedStatus.Equals(SlotStatus.Hung) && statusData.CurrentStatus.Equals(SlotStatus.Hung) == false)
{
if (Prefs.Get <bool>(Preference.EmailReportingEnabled) &&
Prefs.Get <bool>(Preference.ReportHung))
{
SendHungEmail(statusData.ClientName);
}
}
slot.Status = computedStatus;
}
private static SlotStatus DetermineAsyncStatus(LegacyClientStatusData statusData, ILogger logger)
{
#region Get Terminal Time
// Terminal Time - defined as last retrieval time minus twice (7 times for GPU) the current Raw Time per Section.
// if a new frame has not completed in twice the amount of time it should take to complete we should deem this client Hung.
DateTime terminalDateTime;
if (statusData.SlotType.Equals(SlotType.GPU))
{
terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 7));
}
else
{
terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 2));
}
#endregion
#region Determine Unit Progress Value to Use
Debug.Assert(statusData.TimeOfLastUnitStart.Equals(DateTime.MinValue) == false);
DateTime lastProgress = statusData.TimeOfLastUnitStart;
if (statusData.TimeOfLastFrameProgress > statusData.TimeOfLastUnitStart)
{
lastProgress = statusData.TimeOfLastFrameProgress;
}
#endregion
#region Write Verbose Trace
if (logger.IsDebugEnabled)
{
var messages = new List <string>(4);
messages.Add(String.Format(Constants.ClientNameFormat, statusData.ClientName, "DetermineAsyncStatus"));
messages.Add(String.Format(" - Retrieval Time (Date) ------- : {0}", statusData.UnitRetrievalTime));
messages.Add(String.Format(" - Time Of Last Unit Start ----- : {0}", statusData.TimeOfLastUnitStart));
messages.Add(String.Format(" - Time Of Last Frame Progress - : {0}", statusData.TimeOfLastFrameProgress));
messages.Add(String.Format(" - Terminal Time (Date) -------- : {0}", terminalDateTime));
messages.ForEach(logger.Debug);
}
#endregion
if (lastProgress > terminalDateTime)
{
return(SlotStatus.RunningAsync);
}
// time of last progress is less than terminal time
return(SlotStatus.Hung);
}
internal static SlotStatus GetSlotStatus(LegacyClientStatusData statusData, ILogger logger)
{
switch (statusData.ReturnedStatus)
{
case SlotStatus.Running: // at this point, we should not see Running Status
case SlotStatus.RunningAsync: // at this point, we should not see RunningAsync Status
case SlotStatus.RunningNoFrameTimes:
break;
case SlotStatus.Unknown:
logger.ErrorFormat("Unable to Determine Status for Client '{0}'", statusData.ClientName);
// Update Client Status - don't call Determine Status
return(statusData.ReturnedStatus);
case SlotStatus.Offline:
case SlotStatus.Stopped:
case SlotStatus.EuePause:
case SlotStatus.Hung:
case SlotStatus.Paused:
case SlotStatus.SendingWorkPacket:
case SlotStatus.GettingWorkPacket:
// Update Client Status - don't call Determine Status
return(statusData.ReturnedStatus);
}
// if we have a frame time, use it
if (statusData.FrameTime > 0)
{
SlotStatus status = DetermineStatus(statusData, logger);
if (status == SlotStatus.Hung && statusData.AllowRunningAsync)
{
return(DetermineAsyncStatus(statusData, logger));
}
return(status);
}
// no frame time based on the current PPD calculation selection ('LastFrame', 'LastThreeFrames', etc)
// this section attempts to give DetermineStats values to detect Hung clients before they have a valid
// frame time - Issue 10
else
{
// if we have no time stamp
if (statusData.TimeOfLastFrame == TimeSpan.Zero)
{
// use the unit start time
statusData.TimeOfLastFrame = statusData.UnitStartTimeStamp;
}
statusData.FrameTime = GetBaseFrameTime(statusData.BenchmarkAverageFrameTime, statusData.SlotType);
if (DetermineStatus(statusData, logger) == SlotStatus.Hung)
{
// Issue 124
if (statusData.AllowRunningAsync)
{
if (DetermineAsyncStatus(statusData, logger) == SlotStatus.Hung)
{
return(SlotStatus.Hung);
}
return(statusData.ReturnedStatus);
}
return(SlotStatus.Hung);
}
return(statusData.ReturnedStatus);
}
}
private static SlotStatus DetermineStatus(LegacyClientStatusData statusData, ILogger logger)
{
#region Get Terminal Time
// Terminal Time - defined as last retrieval time minus twice (7 times for GPU) the current Raw Time per Section.
// if a new frame has not completed in twice the amount of time it should take to complete we should deem this client Hung.
DateTime terminalDateTime;
if (statusData.SlotType.Equals(SlotType.GPU))
{
terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 7));
}
else
{
terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 2));
}
#endregion
#region Get Last Retrieval Time Date
DateTime currentFrameDateTime;
if (statusData.UtcOffsetIsZero)
{
// get only the date from the last retrieval time (in universal), we'll add the current time below
currentFrameDateTime = new DateTime(statusData.UnitRetrievalTime.Date.Ticks, DateTimeKind.Utc);
}
else
{
// get only the date from the last retrieval time, we'll add the current time below
currentFrameDateTime = statusData.UnitRetrievalTime.Date;
}
#endregion
#region Apply Frame Time Offset and Set Current Frame Time Date
TimeSpan offset = TimeSpan.FromMinutes(statusData.ClientTimeOffset);
TimeSpan adjustedFrameTime = statusData.TimeOfLastFrame;
if (statusData.UtcOffsetIsZero == false)
{
adjustedFrameTime = adjustedFrameTime.Add(statusData.UtcOffset);
}
adjustedFrameTime = adjustedFrameTime.Subtract(offset);
// client time has already rolled over to the next day. the offset correction has
// caused the adjusted frame time span to be negetive. take the that negetive span
// and add it to a full 24 hours to correct.
if (adjustedFrameTime < TimeSpan.Zero)
{
adjustedFrameTime = TimeSpan.FromDays(1).Add(adjustedFrameTime);
}
// the offset correction has caused the frame time span to be greater than 24 hours.
// subtract the extra day from the adjusted frame time span.
else if (adjustedFrameTime > TimeSpan.FromDays(1))
{
adjustedFrameTime = adjustedFrameTime.Subtract(TimeSpan.FromDays(1));
}
// add adjusted Time of Last Frame (TimeSpan) to the DateTime with the correct date
currentFrameDateTime = currentFrameDateTime.Add(adjustedFrameTime);
#endregion
#region Check For Frame from Prior Day (Midnight Rollover on Local Machine)
bool priorDayAdjust = false;
// if the current (and adjusted) frame time hours is greater than the last retrieval time hours,
// and the time difference is greater than an hour, then frame is from the day prior.
// this should only happen after midnight time on the machine running HFM when the monitored client has
// not completed a frame since the local machine time rolled over to the next day, otherwise the time
// stamps between HFM and the client are too far off, a positive offset should be set to correct.
if (currentFrameDateTime.TimeOfDay.Hours > statusData.UnitRetrievalTime.TimeOfDay.Hours &&
currentFrameDateTime.TimeOfDay.Subtract(statusData.UnitRetrievalTime.TimeOfDay).Hours > 0)
{
priorDayAdjust = true;
// subtract 1 day from today's date
currentFrameDateTime = currentFrameDateTime.Subtract(TimeSpan.FromDays(1));
}
#endregion
#region Write Verbose Trace
if (logger.IsDebugEnabled)
{
var messages = new List <string>(10);
messages.Add(String.Format(Constants.ClientNameFormat, statusData.ClientName, "DetermineStatus"));
messages.Add(String.Format(" - Retrieval Time (Date) ------- : {0}", statusData.UnitRetrievalTime));
messages.Add(String.Format(" - Time Of Last Frame (TimeSpan) : {0}", statusData.TimeOfLastFrame));
messages.Add(String.Format(" - Offset (Minutes) ------------ : {0}", statusData.ClientTimeOffset));
messages.Add(String.Format(" - Time Of Last Frame (Adjusted) : {0}", adjustedFrameTime));
messages.Add(String.Format(" - Prior Day Adjustment -------- : {0}", priorDayAdjust));
messages.Add(String.Format(" - Time Of Last Frame (Date) --- : {0}", currentFrameDateTime));
messages.Add(String.Format(" - Terminal Time (Date) -------- : {0}", terminalDateTime));
messages.ForEach(logger.Debug);
}
#endregion
if (currentFrameDateTime > terminalDateTime)
{
return(SlotStatus.Running);
}
else // current frame is less than terminal time
{
return(SlotStatus.Hung);
}
}
