Beispiel #1
0
        private void RaiseDatabaseRedundancyAlertIfNecessary(IADDatabase db, IMonitoringADConfig adConfig)
        {
            DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseRedundancyAlertIfNecessary: DB '{0}': Starting DB-level redundancy checks.", db.Name);
            DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, DatabaseHealthMonitor.NUM_HEALTHY_COPIES_MIN, this.m_statusLookup, adConfig, this.m_propertyUpdateTracker, false);
            IHealthValidationResult     result = databaseRedundancyValidator.Run();

            this.m_dbAlerts.OneRedundantCopy.RaiseAppropriateAlertIfNecessary(result);
            this.m_dbAlerts.TwoCopy.RaiseAppropriateAlertIfNecessary(result);
            this.m_dbAlerts.StaleStatus.RaiseAppropriateAlertIfNecessary(result);
            if (adConfig.Dag.DatacenterActivationMode == DatacenterActivationModeOption.DagOnly)
            {
                this.m_dbAlerts.OneRedundantCopySite.RaiseAppropriateAlertIfNecessary(result);
                return;
            }
            DatabaseHealthMonitor.Tracer.TraceDebug <string, DatacenterActivationModeOption>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseRedundancyAlertIfNecessary: DB '{0}': Skipping raising site alerts because DatacenterActivationMode is '{1}'.", db.Name, adConfig.Dag.DatacenterActivationMode);
        }
Beispiel #2
0
        internal bool DetectLowSpace(IADDatabase db, IMonitoringADConfig adConfig)
        {
            if (db.ReplicationType != ReplicationType.Remote)
            {
                return(false);
            }
            if (!FailedSuspendedCopyAutoReseedWorkflow.IsAutoReseedEnabled(db))
            {
                return(false);
            }
            int num = RegistryParameters.SpaceMonitorMinHealthyCount;
            DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, num, this.m_statusLookup, adConfig, null, true);
            IHealthValidationResult     healthValidationResult      = databaseRedundancyValidator.Run();
            CopyStatusClientCachedEntry targetCopyStatus            = healthValidationResult.TargetCopyStatus;

            if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.Healthy || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
            {
                return(false);
            }
            if (targetCopyStatus.CopyStatus.LastLogInfoIsStale || (targetCopyStatus.CopyStatus.GetCopyQueueLength() < (long)RegistryParameters.SpaceMonitorCopyQueueThreshold && targetCopyStatus.CopyStatus.GetReplayQueueLength() < (long)RegistryParameters.SpaceMonitorReplayQueueThreshold))
            {
                return(false);
            }
            if (healthValidationResult.IsTargetCopyHealthy)
            {
                num++;
            }
            long num2 = (long)(targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1048576UL);
            long num3 = (long)(targetCopyStatus.CopyStatus.DiskTotalSpaceBytes / 1048576UL);
            long num4 = 0L;
            bool flag = false;

            if (num2 <= targetCopyStatus.CopyStatus.GetCopyQueueLength() || num3 <= 0L)
            {
                flag = true;
            }
            else
            {
                num4 = (num2 - targetCopyStatus.CopyStatus.GetCopyQueueLength()) * 1048576L / 1048576L;
                if (num4 <= (long)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB)
                {
                    flag = true;
                }
            }
            SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has {1}MB remaining = {2}MB effective. MinHealthy={3} ReportedHealthy={4}", new object[]
            {
                db.Name,
                num2,
                num4,
                num,
                healthValidationResult.HealthyCopiesCount
            });
            if (!flag)
            {
                this.m_actionSuppression.ReportSuccess(db.Guid);
                return(false);
            }
            bool flag2 = !this.m_actionSuppression.ReportFailure(db.Guid, this.SuppressionWindow);

            SpaceMonitor.Tracer.TraceError <string, bool>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Local copy of Database '{0}' is low on space. Action suppressed = {1}", db.Name, flag2);
            if (flag2)
            {
                return(false);
            }
            ReplayCrimsonEvents.SpaceMonitorLowSpaceDetected.LogPeriodic <string, Guid, long, long, string, long, long, long, long, int, int>(db.Guid, this.m_lowSpaceLoggingPeriod, db.Name, db.Guid, num3, num2, string.Format("{0:0.000}", (double)num2 / (double)num3 * 100.0), targetCopyStatus.CopyStatus.GetCopyQueueLength(), targetCopyStatus.CopyStatus.GetReplayQueueLength(), targetCopyStatus.CopyStatus.LowestLogPresent, targetCopyStatus.CopyStatus.LastLogCopied, healthValidationResult.HealthyCopiesCount, num);
            return(healthValidationResult.HealthyCopiesCount >= num);
        }
Beispiel #3
0
        internal NotificationAction NotifyLowDiskSpace(IADDatabase db, IMonitoringADConfig adConfig)
        {
            DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, 1, this.m_statusLookup, adConfig, null, true);
            IHealthValidationResult     healthValidationResult      = databaseRedundancyValidator.Run();
            CopyStatusClientCachedEntry targetCopyStatus            = healthValidationResult.TargetCopyStatus;

            if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
            {
                return(NotificationAction.None);
            }
            double num   = (double)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB / 1024.0;
            double num2  = targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1024.0 / 1024.0 / 1024.0;
            double num3  = 0.0;
            double num4  = 0.0;
            string text  = SpaceMonitor.FindVolume(new DirectoryInfo(db.EdbFilePath.PathName));
            string text2 = SpaceMonitor.FindVolume(new DirectoryInfo(db.LogFolderPath.PathName));

            if (!string.IsNullOrEmpty(text) && !string.IsNullOrEmpty(text2) && !text.Equals(text2, StringComparison.OrdinalIgnoreCase) && db.EdbFilePath.IsLocalFull)
            {
                ulong num5 = 0UL;
                ulong num6 = 0UL;
                DiskHelper.GetFreeSpace(db.EdbFilePath.PathName, out num5, out num6);
                num3 = num5 / 1024UL / 1024UL / 1024UL;
                num4 = num6 / 1024UL / 1024UL / 1024UL;
            }
            bool   flag  = false;
            string text3 = string.Empty;

            if (num2 <= num)
            {
                text3 = string.Format("'{0}' is low on log volume space [{1}]. Current={2:0.##} GB, Threshold={3:0.##} GB", new object[]
                {
                    db.Name,
                    text2,
                    num2,
                    num
                });
                flag = true;
            }
            if (num3 != 0.0 && num4 <= num)
            {
                text3 += string.Format("{0}'{1}' is low on EDB volume space [{2}]. Current={3:0.##} GB, Threshold={4:0.##} GB", new object[]
                {
                    Environment.NewLine,
                    db.Name,
                    text,
                    num4,
                    num
                });
                flag = true;
            }
            if (flag)
            {
                if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && DateTime.UtcNow - SpaceMonitor.LastNotificationHistory[db.Name] < SpaceMonitor.NotificationFrequency)
                {
                    SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is low on space but notification throttling is hit. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
                    {
                        db.Name,
                        SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? "Never" : SpaceMonitor.LastNotificationHistory[db.Name].ToString(),
                        SpaceMonitor.NotificationFrequency.TotalMinutes,
                        text3
                    });
                    return(NotificationAction.None);
                }
                Exception ex;
                if (BitlockerUtil.IsFilePathOnEncryptingVolume(db.LogFolderPath.PathName, out ex))
                {
                    SpaceMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is currently encrypting. Do not raise alert.", db.Name);
                    return(NotificationAction.None);
                }
                if (ex != null)
                {
                    string text4 = string.Format("IsFilePathOnEncryptingVolume({0}) failed: {1}", db.LogFolderPath.PathName, ex.Message);
                    ReplayCrimsonEvents.BitlockerQueryFailed.LogPeriodic <string, Exception>(Environment.MachineName, DiagCore.DefaultEventSuppressionInterval, text4, ex);
                }
                EventNotificationItem eventNotificationItem = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Critical);
                eventNotificationItem.Publish(false);
                if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name))
                {
                    SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.UtcNow;
                }
                else
                {
                    SpaceMonitor.LastNotificationHistory.Add(db.Name, DateTime.UtcNow);
                }
                return(NotificationAction.RedRaised);
            }
            else
            {
                text3 = string.Format("{0} Status is OK - EdbFreeSpace={1:0.##} GB [{2}], LogFreeSpace={3:0.##} GB [{4}], Threshold={5:0.##} GB", new object[]
                {
                    db.Name,
                    num4,
                    text,
                    num2,
                    text2,
                    num
                });
                if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && SpaceMonitor.LastNotificationHistory[db.Name] != DateTime.MinValue)
                {
                    EventNotificationItem eventNotificationItem2 = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Informational);
                    eventNotificationItem2.Publish(false);
                    SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.MinValue;
                    return(NotificationAction.GreenRaised);
                }
                SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has enough space but red notification was never raised. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
                {
                    db.Name,
                    SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? SpaceMonitor.LastNotificationHistory[db.Name].ToString() : "Never",
                    SpaceMonitor.NotificationFrequency.TotalMinutes,
                    text3
                });
                return(NotificationAction.None);
            }
        }