private void RaiseDatabaseRedundancyAlertIfNecessary(IADDatabase db, IMonitoringADConfig adConfig) { DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseRedundancyAlertIfNecessary: DB '{0}': Starting DB-level redundancy checks.", db.Name); DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, DatabaseHealthMonitor.NUM_HEALTHY_COPIES_MIN, this.m_statusLookup, adConfig, this.m_propertyUpdateTracker, false); IHealthValidationResult result = databaseRedundancyValidator.Run(); this.m_dbAlerts.OneRedundantCopy.RaiseAppropriateAlertIfNecessary(result); this.m_dbAlerts.TwoCopy.RaiseAppropriateAlertIfNecessary(result); this.m_dbAlerts.StaleStatus.RaiseAppropriateAlertIfNecessary(result); if (adConfig.Dag.DatacenterActivationMode == DatacenterActivationModeOption.DagOnly) { this.m_dbAlerts.OneRedundantCopySite.RaiseAppropriateAlertIfNecessary(result); return; } DatabaseHealthMonitor.Tracer.TraceDebug <string, DatacenterActivationModeOption>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseRedundancyAlertIfNecessary: DB '{0}': Skipping raising site alerts because DatacenterActivationMode is '{1}'.", db.Name, adConfig.Dag.DatacenterActivationMode); }
internal bool DetectLowSpace(IADDatabase db, IMonitoringADConfig adConfig) { if (db.ReplicationType != ReplicationType.Remote) { return(false); } if (!FailedSuspendedCopyAutoReseedWorkflow.IsAutoReseedEnabled(db)) { return(false); } int num = RegistryParameters.SpaceMonitorMinHealthyCount; DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, num, this.m_statusLookup, adConfig, null, true); IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run(); CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus; if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.Healthy || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL) { return(false); } if (targetCopyStatus.CopyStatus.LastLogInfoIsStale || (targetCopyStatus.CopyStatus.GetCopyQueueLength() < (long)RegistryParameters.SpaceMonitorCopyQueueThreshold && targetCopyStatus.CopyStatus.GetReplayQueueLength() < (long)RegistryParameters.SpaceMonitorReplayQueueThreshold)) { return(false); } if (healthValidationResult.IsTargetCopyHealthy) { num++; } long num2 = (long)(targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1048576UL); long num3 = (long)(targetCopyStatus.CopyStatus.DiskTotalSpaceBytes / 1048576UL); long num4 = 0L; bool flag = false; if (num2 <= targetCopyStatus.CopyStatus.GetCopyQueueLength() || num3 <= 0L) { flag = true; } else { num4 = (num2 - targetCopyStatus.CopyStatus.GetCopyQueueLength()) * 1048576L / 1048576L; if (num4 <= (long)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB) { flag = true; } } SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has {1}MB remaining = {2}MB effective. MinHealthy={3} ReportedHealthy={4}", new object[] { db.Name, num2, num4, num, healthValidationResult.HealthyCopiesCount }); if (!flag) { this.m_actionSuppression.ReportSuccess(db.Guid); return(false); } bool flag2 = !this.m_actionSuppression.ReportFailure(db.Guid, this.SuppressionWindow); SpaceMonitor.Tracer.TraceError <string, bool>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Local copy of Database '{0}' is low on space. Action suppressed = {1}", db.Name, flag2); if (flag2) { return(false); } ReplayCrimsonEvents.SpaceMonitorLowSpaceDetected.LogPeriodic <string, Guid, long, long, string, long, long, long, long, int, int>(db.Guid, this.m_lowSpaceLoggingPeriod, db.Name, db.Guid, num3, num2, string.Format("{0:0.000}", (double)num2 / (double)num3 * 100.0), targetCopyStatus.CopyStatus.GetCopyQueueLength(), targetCopyStatus.CopyStatus.GetReplayQueueLength(), targetCopyStatus.CopyStatus.LowestLogPresent, targetCopyStatus.CopyStatus.LastLogCopied, healthValidationResult.HealthyCopiesCount, num); return(healthValidationResult.HealthyCopiesCount >= num); }
internal NotificationAction NotifyLowDiskSpace(IADDatabase db, IMonitoringADConfig adConfig) { DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, 1, this.m_statusLookup, adConfig, null, true); IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run(); CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus; if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL) { return(NotificationAction.None); } double num = (double)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB / 1024.0; double num2 = targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1024.0 / 1024.0 / 1024.0; double num3 = 0.0; double num4 = 0.0; string text = SpaceMonitor.FindVolume(new DirectoryInfo(db.EdbFilePath.PathName)); string text2 = SpaceMonitor.FindVolume(new DirectoryInfo(db.LogFolderPath.PathName)); if (!string.IsNullOrEmpty(text) && !string.IsNullOrEmpty(text2) && !text.Equals(text2, StringComparison.OrdinalIgnoreCase) && db.EdbFilePath.IsLocalFull) { ulong num5 = 0UL; ulong num6 = 0UL; DiskHelper.GetFreeSpace(db.EdbFilePath.PathName, out num5, out num6); num3 = num5 / 1024UL / 1024UL / 1024UL; num4 = num6 / 1024UL / 1024UL / 1024UL; } bool flag = false; string text3 = string.Empty; if (num2 <= num) { text3 = string.Format("'{0}' is low on log volume space [{1}]. Current={2:0.##} GB, Threshold={3:0.##} GB", new object[] { db.Name, text2, num2, num }); flag = true; } if (num3 != 0.0 && num4 <= num) { text3 += string.Format("{0}'{1}' is low on EDB volume space [{2}]. Current={3:0.##} GB, Threshold={4:0.##} GB", new object[] { Environment.NewLine, db.Name, text, num4, num }); flag = true; } if (flag) { if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && DateTime.UtcNow - SpaceMonitor.LastNotificationHistory[db.Name] < SpaceMonitor.NotificationFrequency) { SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is low on space but notification throttling is hit. LastNotify={1}, Throttling={2}mins, Message={3}", new object[] { db.Name, SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? "Never" : SpaceMonitor.LastNotificationHistory[db.Name].ToString(), SpaceMonitor.NotificationFrequency.TotalMinutes, text3 }); return(NotificationAction.None); } Exception ex; if (BitlockerUtil.IsFilePathOnEncryptingVolume(db.LogFolderPath.PathName, out ex)) { SpaceMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is currently encrypting. Do not raise alert.", db.Name); return(NotificationAction.None); } if (ex != null) { string text4 = string.Format("IsFilePathOnEncryptingVolume({0}) failed: {1}", db.LogFolderPath.PathName, ex.Message); ReplayCrimsonEvents.BitlockerQueryFailed.LogPeriodic <string, Exception>(Environment.MachineName, DiagCore.DefaultEventSuppressionInterval, text4, ex); } EventNotificationItem eventNotificationItem = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Critical); eventNotificationItem.Publish(false); if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name)) { SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.UtcNow; } else { SpaceMonitor.LastNotificationHistory.Add(db.Name, DateTime.UtcNow); } return(NotificationAction.RedRaised); } else { text3 = string.Format("{0} Status is OK - EdbFreeSpace={1:0.##} GB [{2}], LogFreeSpace={3:0.##} GB [{4}], Threshold={5:0.##} GB", new object[] { db.Name, num4, text, num2, text2, num }); if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && SpaceMonitor.LastNotificationHistory[db.Name] != DateTime.MinValue) { EventNotificationItem eventNotificationItem2 = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Informational); eventNotificationItem2.Publish(false); SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.MinValue; return(NotificationAction.GreenRaised); } SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has enough space but red notification was never raised. LastNotify={1}, Throttling={2}mins, Message={3}", new object[] { db.Name, SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? SpaceMonitor.LastNotificationHistory[db.Name].ToString() : "Never", SpaceMonitor.NotificationFrequency.TotalMinutes, text3 }); return(NotificationAction.None); } }