public void StatusTestSet1_Running() { DateTime date = DateTime.Now.Date; var statusData = new StatusData(); statusData.ClientName = "Status Test"; statusData.SlotType = SlotType.CPU; statusData.ClientTimeOffset = 0; statusData.UtcOffsetIsZero = false; statusData.UtcOffset = TimeSpan.Zero; statusData.AllowRunningAsync = false; statusData.CurrentStatus = SlotStatus.Running; statusData.ReturnedStatus = SlotStatus.RunningNoFrameTimes; statusData.UnitRetrievalTime = date.Add(new TimeSpan(3, 0, 0)); statusData.UnitStartTimeStamp = new TimeSpan(1, 55, 0); statusData.TimeOfLastFrame = new TimeSpan(2, 50, 0); statusData.TimeOfLastUnitStart = date.Add(new TimeSpan(2, 0, 0)); statusData.TimeOfLastFrameProgress = date.Add(new TimeSpan(3, 0, 0)); statusData.FrameTime = 750; statusData.BenchmarkAverageFrameTime = new TimeSpan(0, 12, 35); Assert.AreEqual(SlotStatus.Running, _statusLogic.HandleStatusData(statusData)); }
public void StatusTestSet3_Hung_NoAsync() { DateTime date = DateTime.Now.Date; var statusData = new StatusData(); statusData.ClientName = "Status Test"; statusData.SlotType = SlotType.CPU; statusData.ClientTimeOffset = 0; statusData.UtcOffsetIsZero = false; statusData.UtcOffset = TimeSpan.Zero; statusData.AllowRunningAsync = false; statusData.CurrentStatus = SlotStatus.Hung; statusData.ReturnedStatus = SlotStatus.RunningNoFrameTimes; statusData.UnitRetrievalTime = date.Add(new TimeSpan(12, 0, 0)); // Client Clock is 4 Hours Behind this machine statusData.UnitStartTimeStamp = new TimeSpan(6, 0, 0); statusData.TimeOfLastFrame = new TimeSpan(7, 50, 0); statusData.TimeOfLastUnitStart = date.Add(new TimeSpan(10, 0, 0)); statusData.TimeOfLastFrameProgress = date.Add(new TimeSpan(11, 50, 0)); statusData.FrameTime = 633; // 10 Minutes 33 Seconds statusData.BenchmarkAverageFrameTime = new TimeSpan(0, 10, 25); statusData.AllowRunningAsync = false; Assert.AreEqual(SlotStatus.Hung, _statusLogic.HandleStatusData(statusData)); }
public void StatusTestSet2_Hung_NoAsync() { DateTime date = DateTime.Now.Date; var statusData = new StatusData(); statusData.ClientName = "Status Test"; statusData.SlotType = SlotType.CPU; statusData.ClientTimeOffset = 0; statusData.UtcOffsetIsZero = false; statusData.UtcOffset = TimeSpan.Zero; statusData.AllowRunningAsync = false; statusData.CurrentStatus = SlotStatus.GettingWorkPacket; statusData.ReturnedStatus = SlotStatus.RunningNoFrameTimes; statusData.UnitRetrievalTime = date.Add(new TimeSpan(3, 0, 0)); // Client Clock is ~2 Hours Behind this machine statusData.UnitStartTimeStamp = new TimeSpan(0, 55, 0); statusData.TimeOfLastFrame = TimeSpan.Zero; statusData.TimeOfLastUnitStart = date.Add(new TimeSpan(3, 0, 0)); statusData.TimeOfLastFrameProgress = DateTime.MinValue; statusData.FrameTime = 0; statusData.BenchmarkAverageFrameTime = new TimeSpan(0, 12, 35); statusData.AllowRunningAsync = false; Assert.AreEqual(SlotStatus.Hung, _statusLogic.HandleStatusData(statusData)); }
/// <summary> /// Handles the Client Status Returned by Log Parsing and then determine the Status. /// </summary> /// <param name="statusData">Client Status Data</param> public SlotStatus HandleStatusData(StatusData statusData) { switch (statusData.ReturnedStatus) { case SlotStatus.Running: // at this point, we should not see Running Status case SlotStatus.RunningAsync: // at this point, we should not see RunningAsync Status case SlotStatus.RunningNoFrameTimes: break; case SlotStatus.Unknown: _logger.ErrorFormat("Unable to Determine Status for Client '{0}'", statusData.ClientName); // Update Client Status - don't call Determine Status return statusData.ReturnedStatus; case SlotStatus.Offline: case SlotStatus.Stopped: case SlotStatus.EuePause: case SlotStatus.Hung: case SlotStatus.Paused: case SlotStatus.SendingWorkPacket: case SlotStatus.GettingWorkPacket: // Update Client Status - don't call Determine Status return statusData.ReturnedStatus; } // if we have a frame time, use it if (statusData.FrameTime > 0) { SlotStatus status = DetermineStatus(statusData); if (status.Equals(SlotStatus.Hung) && statusData.AllowRunningAsync) // Issue 124 { return DetermineAsyncStatus(statusData); } return status; } // no frame time based on the current PPD calculation selection ('LastFrame', 'LastThreeFrames', etc) // this section attempts to give DetermineStats values to detect Hung clients before they have a valid // frame time - Issue 10 else { // if we have no time stamp if (statusData.TimeOfLastFrame == TimeSpan.Zero) { // use the unit start time statusData.TimeOfLastFrame = statusData.UnitStartTimeStamp; } statusData.FrameTime = GetBaseFrameTime(statusData.BenchmarkAverageFrameTime, statusData.SlotType); if (DetermineStatus(statusData).Equals(SlotStatus.Hung)) { // Issue 124 if (statusData.AllowRunningAsync) { if (DetermineAsyncStatus(statusData).Equals(SlotStatus.Hung)) { return SlotStatus.Hung; } else { return statusData.ReturnedStatus; } } return SlotStatus.Hung; } else { return statusData.ReturnedStatus; } } }
/// <summary> /// Determine Client Status /// </summary> /// <param name="statusData">Client Status Data</param> private SlotStatus DetermineAsyncStatus(StatusData statusData) { #region Get Terminal Time // Terminal Time - defined as last retrieval time minus twice (7 times for GPU) the current Raw Time per Section. // if a new frame has not completed in twice the amount of time it should take to complete we should deem this client Hung. DateTime terminalDateTime; if (statusData.SlotType.Equals(SlotType.GPU)) { terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 7)); } else { terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 2)); } #endregion #region Determine Unit Progress Value to Use Debug.Assert(statusData.TimeOfLastUnitStart.Equals(DateTime.MinValue) == false); DateTime lastProgress = statusData.TimeOfLastUnitStart; if (statusData.TimeOfLastFrameProgress > statusData.TimeOfLastUnitStart) { lastProgress = statusData.TimeOfLastFrameProgress; } #endregion #region Write Verbose Trace if (_logger.IsDebugEnabled) { var messages = new List<string>(4); messages.Add(String.Format("{0} ({1})", Instrumentation.FunctionName, statusData.ClientName)); messages.Add(String.Format(" - Retrieval Time (Date) ------- : {0}", statusData.UnitRetrievalTime)); messages.Add(String.Format(" - Time Of Last Unit Start ----- : {0}", statusData.TimeOfLastUnitStart)); messages.Add(String.Format(" - Time Of Last Frame Progress - : {0}", statusData.TimeOfLastFrameProgress)); messages.Add(String.Format(" - Terminal Time (Date) -------- : {0}", terminalDateTime)); messages.ForEach(x => _logger.Debug(x)); } #endregion if (lastProgress > terminalDateTime) { return SlotStatus.RunningAsync; } else // time of last progress is less than terminal time { return SlotStatus.Hung; } }
/// <summary> /// Determine Client Status /// </summary> /// <param name="statusData">Client Status Data</param> private SlotStatus DetermineStatus(StatusData statusData) { #region Get Terminal Time // Terminal Time - defined as last retrieval time minus twice (7 times for GPU) the current Raw Time per Section. // if a new frame has not completed in twice the amount of time it should take to complete we should deem this client Hung. DateTime terminalDateTime; if (statusData.SlotType.Equals(SlotType.GPU)) { terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 7)); } else { terminalDateTime = statusData.UnitRetrievalTime.Subtract(TimeSpan.FromSeconds(statusData.FrameTime * 2)); } #endregion #region Get Last Retrieval Time Date DateTime currentFrameDateTime; if (statusData.UtcOffsetIsZero) { // get only the date from the last retrieval time (in universal), we'll add the current time below currentFrameDateTime = new DateTime(statusData.UnitRetrievalTime.Date.Ticks, DateTimeKind.Utc); } else { // get only the date from the last retrieval time, we'll add the current time below currentFrameDateTime = statusData.UnitRetrievalTime.Date; } #endregion #region Apply Frame Time Offset and Set Current Frame Time Date TimeSpan offset = TimeSpan.FromMinutes(statusData.ClientTimeOffset); TimeSpan adjustedFrameTime = statusData.TimeOfLastFrame; if (statusData.UtcOffsetIsZero == false) { adjustedFrameTime = adjustedFrameTime.Add(statusData.UtcOffset); } adjustedFrameTime = adjustedFrameTime.Subtract(offset); // client time has already rolled over to the next day. the offset correction has // caused the adjusted frame time span to be negetive. take the that negetive span // and add it to a full 24 hours to correct. if (adjustedFrameTime < TimeSpan.Zero) { adjustedFrameTime = TimeSpan.FromDays(1).Add(adjustedFrameTime); } // the offset correction has caused the frame time span to be greater than 24 hours. // subtract the extra day from the adjusted frame time span. else if (adjustedFrameTime > TimeSpan.FromDays(1)) { adjustedFrameTime = adjustedFrameTime.Subtract(TimeSpan.FromDays(1)); } // add adjusted Time of Last Frame (TimeSpan) to the DateTime with the correct date currentFrameDateTime = currentFrameDateTime.Add(adjustedFrameTime); #endregion #region Check For Frame from Prior Day (Midnight Rollover on Local Machine) bool priorDayAdjust = false; // if the current (and adjusted) frame time hours is greater than the last retrieval time hours, // and the time difference is greater than an hour, then frame is from the day prior. // this should only happen after midnight time on the machine running HFM when the monitored client has // not completed a frame since the local machine time rolled over to the next day, otherwise the time // stamps between HFM and the client are too far off, a positive offset should be set to correct. if (currentFrameDateTime.TimeOfDay.Hours > statusData.UnitRetrievalTime.TimeOfDay.Hours && currentFrameDateTime.TimeOfDay.Subtract(statusData.UnitRetrievalTime.TimeOfDay).Hours > 0) { priorDayAdjust = true; // subtract 1 day from today's date currentFrameDateTime = currentFrameDateTime.Subtract(TimeSpan.FromDays(1)); } #endregion #region Write Verbose Trace if (_logger.IsDebugEnabled) { var messages = new List<string>(10); messages.Add(String.Format("{0} ({1})", Instrumentation.FunctionName, statusData.ClientName)); messages.Add(String.Format(" - Retrieval Time (Date) ------- : {0}", statusData.UnitRetrievalTime)); messages.Add(String.Format(" - Time Of Last Frame (TimeSpan) : {0}", statusData.TimeOfLastFrame)); messages.Add(String.Format(" - Offset (Minutes) ------------ : {0}", statusData.ClientTimeOffset)); messages.Add(String.Format(" - Time Of Last Frame (Adjusted) : {0}", adjustedFrameTime)); messages.Add(String.Format(" - Prior Day Adjustment -------- : {0}", priorDayAdjust)); messages.Add(String.Format(" - Time Of Last Frame (Date) --- : {0}", currentFrameDateTime)); messages.Add(String.Format(" - Terminal Time (Date) -------- : {0}", terminalDateTime)); messages.ForEach(x => _logger.Debug(x)); } #endregion if (currentFrameDateTime > terminalDateTime) { return SlotStatus.Running; } else // current frame is less than terminal time { return SlotStatus.Hung; } }
/// <summary> /// Handles the Client Status Returned by Log Parsing and then determines what values to feed the DetermineStatus routine. /// </summary> private void HandleReturnedStatus(SlotStatus returnedStatus, SlotModel slot) { var statusData = new StatusData { ClientName = Settings.Name, SlotType = slot.UnitInfoLogic.UnitInfoData.SlotType, UnitRetrievalTime = slot.UnitInfoLogic.UnitInfoData.UnitRetrievalTime, UtcOffsetIsZero = Settings.UtcOffsetIsZero, UtcOffset = TimeZone.CurrentTimeZone.GetUtcOffset(DateTime.Now), ClientTimeOffset = Settings.ClientTimeOffset, TimeOfLastUnitStart = slot.TimeOfLastUnitStart, TimeOfLastFrameProgress = slot.TimeOfLastFrameProgress, CurrentStatus = slot.Status, ReturnedStatus = returnedStatus, FrameTime = slot.UnitInfoLogic.GetRawTime(Prefs.Get<PpdCalculationType>(Preference.PpdCalculation)), BenchmarkAverageFrameTime = GetBenchmarkAverageFrameTimeOrDefault(slot.UnitInfo), TimeOfLastFrame = slot.UnitInfoLogic.UnitInfoData.CurrentFrame == null ? TimeSpan.Zero : slot.UnitInfoLogic.UnitInfoData.CurrentFrame.TimeOfFrame, UnitStartTimeStamp = slot.UnitInfoLogic.UnitInfoData.UnitStartTimeStamp, AllowRunningAsync = Prefs.Get<bool>(Preference.AllowRunningAsync) }; SlotStatus computedStatus = StatusLogic.HandleStatusData(statusData); // If the returned status is EuePause and current status is not if (computedStatus.Equals(SlotStatus.EuePause) && statusData.CurrentStatus.Equals(SlotStatus.EuePause) == false) { if (Prefs.Get<bool>(Preference.EmailReportingEnabled) && Prefs.Get<bool>(Preference.ReportEuePause)) { SendEuePauseEmail(statusData.ClientName); } } // If the returned status is Hung and current status is not if (computedStatus.Equals(SlotStatus.Hung) && statusData.CurrentStatus.Equals(SlotStatus.Hung) == false) { if (Prefs.Get<bool>(Preference.EmailReportingEnabled) && Prefs.Get<bool>(Preference.ReportHung)) { SendHungEmail(statusData.ClientName); } } slot.Status = computedStatus; }