예제 #1
0
        private void ProcessDatabaseInternal(IADDatabase db, IMonitoringADConfig adConfig)
        {
            if (db.ReplicationType != ReplicationType.Remote)
            {
                ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Database '{0}' is not a replicated database. Skipping this database.", db.Name);
                return;
            }
            IADDatabaseCopy  databaseCopy  = this.GetDatabaseCopy(db, AmServerName.LocalComputerName);
            EnhancedTimeSpan replayLagTime = databaseCopy.ReplayLagTime;

            if (replayLagTime == EnhancedTimeSpan.Zero)
            {
                ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' is not a lag copy. Skipping this database.", db.Name);
                return;
            }
            ReplayLagManager.Tracer.TraceDebug <string, EnhancedTimeSpan>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' is a lag copy. Configured lag: {1}. Processing this database.", db.Name, replayLagTime);
            DatabaseAvailabilityValidator databaseAvailabilityValidator = new DatabaseAvailabilityValidator(db, ReplayLagManager.NUM_AVAILABLE_NONLAG_COPIES_MIN, this.m_statusLookup, adConfig, null, true);
            IHealthValidationResult       healthValidationResult        = databaseAvailabilityValidator.Run();
            int num = ReplayLagManager.NUM_AVAILABLE_NONLAG_COPIES_MIN;

            if (healthValidationResult.IsTargetCopyHealthy)
            {
                num++;
                ReplayLagManager.Tracer.TraceDebug <string, int>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local lag-copy of Database '{0}' is available. Increasing the minimum available copy count to: {1}", db.Name, num);
            }
            CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus;

            if (healthValidationResult.HealthyCopiesCount >= num)
            {
                ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' has met availability critera. Replay Lag will be re-instated as necessary.", db.Name);
                if (this.m_errorSuppression.ReportSuccess(db.Guid, ReplayLagManager.EnableLagSuppressionWindow))
                {
                    this.EnableReplayLag(db, targetCopyStatus);
                    return;
                }
                ReplayLagManager.Tracer.TraceDebug <string, double>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: '{0}': Skipping EnableReplayLag() due to transient suppression of {1} secs", db.Name, ReplayLagManager.EnableLagSuppressionWindow.TotalSeconds);
                return;
            }
            else
            {
                ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' has *NOT* met availability critera. Replay Lag will be played down as necessary.", db.Name);
                string errorMessageWithoutFullStatus = healthValidationResult.ErrorMessageWithoutFullStatus;
                if (this.m_errorSuppression.ReportFailure(db.Guid, ReplayLagManager.DisableLagSuppressionWindow))
                {
                    this.DisableReplayLag(db, targetCopyStatus, errorMessageWithoutFullStatus);
                    return;
                }
                ReplayLagManager.Tracer.TraceDebug <string, double>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: '{0}': Skipping DisableReplayLag() due to transient suppression of {1} secs", db.Name, ReplayLagManager.DisableLagSuppressionWindow.TotalSeconds);
                ReplayCrimsonEvents.RLMDisableReplayLagRequestSuppressed.LogPeriodic <string, string, Guid, string, TimeSpan>(Environment.MachineName, DateTimeHelper.FourHours, db.Name, Environment.MachineName, db.Guid, errorMessageWithoutFullStatus, ReplayLagManager.DisableLagSuppressionWindow);
                return;
            }
        }
예제 #2
0
        private void RaiseDatabaseAvailabilityAlertIfNecessary(IADDatabase db, IMonitoringADConfig adConfig)
        {
            DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseAvailabilityAlertIfNecessary: DB '{0}': Starting DB-level availability checks.", db.Name);
            DatabaseAvailabilityValidator databaseAvailabilityValidator = new DatabaseAvailabilityValidator(db, DatabaseHealthMonitor.NUM_HEALTHY_COPIES_MIN, this.m_statusLookup, adConfig, this.m_propertyUpdateTracker, false);
            IHealthValidationResult       result = databaseAvailabilityValidator.Run();

            this.m_dbAlerts.OneAvailableCopy.RaiseAppropriateAlertIfNecessary(result);
            if (adConfig.Dag.DatacenterActivationMode == DatacenterActivationModeOption.DagOnly)
            {
                this.m_dbAlerts.OneAvailableCopySite.RaiseAppropriateAlertIfNecessary(result);
                return;
            }
            DatabaseHealthMonitor.Tracer.TraceDebug <string, DatacenterActivationModeOption>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseAvailabilityAlertIfNecessary: DB '{0}': Skipping raising site alerts because DatacenterActivationMode is '{1}'.", db.Name, adConfig.Dag.DatacenterActivationMode);
        }
예제 #3
0
        private static string GetActiveServerName(IHealthValidationResultMinimal validationResult)
        {
            IHealthValidationResult healthValidationResult = (IHealthValidationResult)validationResult;
            string result = null;

            if (healthValidationResult.ActiveCopyStatus != null)
            {
                result = healthValidationResult.ActiveCopyStatus.ServerContacted.NetbiosName;
            }
            else if (healthValidationResult.TargetCopyStatus != null && healthValidationResult.TargetCopyStatus.ActiveServer != null)
            {
                result = healthValidationResult.TargetCopyStatus.ActiveServer.NetbiosName;
            }
            return(result);
        }
예제 #4
0
        private void RaiseDatabasePotentialRedundancyAlertIfNecessary(IADDatabase db, IMonitoringADConfig adConfig)
        {
            if (this.stopWatch.Elapsed.TotalSeconds < (double)RegistryParameters.DatabaseHealthCheckDelayInRaisingDatabasePotentialRedundancyAlertDueToServiceStartInSec)
            {
                DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabasePotentialRedundancyAlertIfNecessary: Service just started, skip DB-level checks.", db.Name);
                return;
            }
            DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabasePotentialRedundancyAlertIfNecessary: DB '{0}': Starting DB-level checks.", db.Name);
            DatabaseReplicationNotStalledValidator databaseReplicationNotStalledValidator = new DatabaseReplicationNotStalledValidator(db, this.m_statusLookup, adConfig, this.m_propertyUpdateTracker, false);
            IHealthValidationResult healthValidationResult = databaseReplicationNotStalledValidator.Run();
            int databaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin = RegistryParameters.DatabaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin;

            if (!healthValidationResult.IsValidationSuccessful && healthValidationResult.TotalPassiveCopiesCount < databaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin)
            {
                DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabasePotentialRedundancyAlertIfNecessary: DB '{0}': Skipping raising potential redundancy alert because not enough passive copies are found", db.Name);
                string text = string.Format("{0}-NotEnoughTotalPassiveCopies", healthValidationResult.Identity);
                ReplayCrimsonEvents.DatabaseLevelPotentialRedundancyCheckFailedButNotEnoughTotalPassiveCopies.LogPeriodic <string, int, int, int, int, string, Guid>(text, TimeSpan.FromMinutes(60.0), healthValidationResult.Identity, healthValidationResult.HealthyCopiesCount, healthValidationResult.HealthyPassiveCopiesCount, healthValidationResult.TotalPassiveCopiesCount, databaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin, "", healthValidationResult.IdentityGuid);
                return;
            }
            this.m_dbAlerts.PotentialOneRedundantCopy.RaiseAppropriateAlertIfNecessary(healthValidationResult);
        }
예제 #5
0
        private void RunLogic(bool isFastLagPlaydownDesired, long replayQ)
        {
            if (RegistryParameters.DisableDatabaseScan)
            {
                this.isEnabled = false;
                return;
            }
            if (isFastLagPlaydownDesired)
            {
                if (this.DisableScan())
                {
                    this.LogDisable("FastLagPlaydownDesired");
                }
                return;
            }
            if (!this.isEnabled && ExDateTime.UtcNow - this.lastDisableTime < this.Parameters.MinDisabledWindow)
            {
                return;
            }
            if (this.isEnabled && replayQ < this.Parameters.MinReplayQOfInterest)
            {
                return;
            }
            if (this.isLagCopy)
            {
                if (replayQ > this.Parameters.MaxReplayQForBehindCopy)
                {
                    if (this.DisableScan())
                    {
                        this.LogDisable(string.Format("Lag too behind. ReplayQ={0} MaxQ={1}", replayQ, this.Parameters.MaxReplayQForBehindCopy));
                        return;
                    }
                }
                else if (!this.isEnabled && replayQ < this.Parameters.MaxReplayQForAvailableCopy && this.EnableScan())
                {
                    this.LogEnable(string.Format("Lag catching up. ReplayQ={0} MaxQ={1}", replayQ, this.Parameters.MaxReplayQForAvailableCopy));
                }
                return;
            }
            bool flag = this.sensor.DatabaseHasSufficientAvailability();

            if (this.sensor.FailedToRead)
            {
                if (replayQ > this.Parameters.MaxReplayQForAvailableCopy && this.DisableScan())
                {
                    this.LogDisable("DatabaseHasSufficientAvailability failed");
                }
                return;
            }
            IHealthValidationResult lastReading = this.sensor.LastReading;

            if (!flag)
            {
                if (this.DisableScan())
                {
                    string reason = string.Format("Database '{0}' has insufficient availability. MinAvail={1} CurAvail={2}", this.Database.Name, this.sensor.MinAvailableCopies, lastReading.HealthyCopiesCount);
                    this.LogDisable(reason);
                }
                return;
            }
            if (this.isEnabled)
            {
                if (replayQ <= this.Parameters.MaxReplayQForAvailableCopy)
                {
                    return;
                }
                if (lastReading.IsTargetCopyHealthy)
                {
                    if (this.DisableScan())
                    {
                        this.LogDisable(string.Format("Availability at risk. ReplayQ={0}", replayQ));
                    }
                    return;
                }
                if (replayQ > this.Parameters.MaxReplayQForBehindCopy || (this.keepingUp != null && this.keepingUp.ReplayQueueNotKeepingUp > 0L))
                {
                    if (this.DisableScan())
                    {
                        this.LogDisable(string.Format("Too far behind. ReplayQ={0}", replayQ));
                    }
                    return;
                }
            }
            else if (lastReading.IsTargetCopyHealthy)
            {
                if (replayQ > this.Parameters.MinReplayQOfInterest)
                {
                    return;
                }
                if (this.EnableScan())
                {
                    this.LogEnable(string.Format("Available copy recovered. ReplayQ={0}", replayQ));
                }
                return;
            }
            else if (this.EnableScan())
            {
                this.LogEnable(string.Format("Availability is ok due to other copies. This copy can resume scanning. ReplayQ={0}", replayQ));
            }
        }
예제 #6
0
        internal bool DetectLowSpace(IADDatabase db, IMonitoringADConfig adConfig)
        {
            if (db.ReplicationType != ReplicationType.Remote)
            {
                return(false);
            }
            if (!FailedSuspendedCopyAutoReseedWorkflow.IsAutoReseedEnabled(db))
            {
                return(false);
            }
            int num = RegistryParameters.SpaceMonitorMinHealthyCount;
            DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, num, this.m_statusLookup, adConfig, null, true);
            IHealthValidationResult     healthValidationResult      = databaseRedundancyValidator.Run();
            CopyStatusClientCachedEntry targetCopyStatus            = healthValidationResult.TargetCopyStatus;

            if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.Healthy || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
            {
                return(false);
            }
            if (targetCopyStatus.CopyStatus.LastLogInfoIsStale || (targetCopyStatus.CopyStatus.GetCopyQueueLength() < (long)RegistryParameters.SpaceMonitorCopyQueueThreshold && targetCopyStatus.CopyStatus.GetReplayQueueLength() < (long)RegistryParameters.SpaceMonitorReplayQueueThreshold))
            {
                return(false);
            }
            if (healthValidationResult.IsTargetCopyHealthy)
            {
                num++;
            }
            long num2 = (long)(targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1048576UL);
            long num3 = (long)(targetCopyStatus.CopyStatus.DiskTotalSpaceBytes / 1048576UL);
            long num4 = 0L;
            bool flag = false;

            if (num2 <= targetCopyStatus.CopyStatus.GetCopyQueueLength() || num3 <= 0L)
            {
                flag = true;
            }
            else
            {
                num4 = (num2 - targetCopyStatus.CopyStatus.GetCopyQueueLength()) * 1048576L / 1048576L;
                if (num4 <= (long)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB)
                {
                    flag = true;
                }
            }
            SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has {1}MB remaining = {2}MB effective. MinHealthy={3} ReportedHealthy={4}", new object[]
            {
                db.Name,
                num2,
                num4,
                num,
                healthValidationResult.HealthyCopiesCount
            });
            if (!flag)
            {
                this.m_actionSuppression.ReportSuccess(db.Guid);
                return(false);
            }
            bool flag2 = !this.m_actionSuppression.ReportFailure(db.Guid, this.SuppressionWindow);

            SpaceMonitor.Tracer.TraceError <string, bool>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Local copy of Database '{0}' is low on space. Action suppressed = {1}", db.Name, flag2);
            if (flag2)
            {
                return(false);
            }
            ReplayCrimsonEvents.SpaceMonitorLowSpaceDetected.LogPeriodic <string, Guid, long, long, string, long, long, long, long, int, int>(db.Guid, this.m_lowSpaceLoggingPeriod, db.Name, db.Guid, num3, num2, string.Format("{0:0.000}", (double)num2 / (double)num3 * 100.0), targetCopyStatus.CopyStatus.GetCopyQueueLength(), targetCopyStatus.CopyStatus.GetReplayQueueLength(), targetCopyStatus.CopyStatus.LowestLogPresent, targetCopyStatus.CopyStatus.LastLogCopied, healthValidationResult.HealthyCopiesCount, num);
            return(healthValidationResult.HealthyCopiesCount >= num);
        }
예제 #7
0
        internal NotificationAction NotifyLowDiskSpace(IADDatabase db, IMonitoringADConfig adConfig)
        {
            DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, 1, this.m_statusLookup, adConfig, null, true);
            IHealthValidationResult     healthValidationResult      = databaseRedundancyValidator.Run();
            CopyStatusClientCachedEntry targetCopyStatus            = healthValidationResult.TargetCopyStatus;

            if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
            {
                return(NotificationAction.None);
            }
            double num   = (double)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB / 1024.0;
            double num2  = targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1024.0 / 1024.0 / 1024.0;
            double num3  = 0.0;
            double num4  = 0.0;
            string text  = SpaceMonitor.FindVolume(new DirectoryInfo(db.EdbFilePath.PathName));
            string text2 = SpaceMonitor.FindVolume(new DirectoryInfo(db.LogFolderPath.PathName));

            if (!string.IsNullOrEmpty(text) && !string.IsNullOrEmpty(text2) && !text.Equals(text2, StringComparison.OrdinalIgnoreCase) && db.EdbFilePath.IsLocalFull)
            {
                ulong num5 = 0UL;
                ulong num6 = 0UL;
                DiskHelper.GetFreeSpace(db.EdbFilePath.PathName, out num5, out num6);
                num3 = num5 / 1024UL / 1024UL / 1024UL;
                num4 = num6 / 1024UL / 1024UL / 1024UL;
            }
            bool   flag  = false;
            string text3 = string.Empty;

            if (num2 <= num)
            {
                text3 = string.Format("'{0}' is low on log volume space [{1}]. Current={2:0.##} GB, Threshold={3:0.##} GB", new object[]
                {
                    db.Name,
                    text2,
                    num2,
                    num
                });
                flag = true;
            }
            if (num3 != 0.0 && num4 <= num)
            {
                text3 += string.Format("{0}'{1}' is low on EDB volume space [{2}]. Current={3:0.##} GB, Threshold={4:0.##} GB", new object[]
                {
                    Environment.NewLine,
                    db.Name,
                    text,
                    num4,
                    num
                });
                flag = true;
            }
            if (flag)
            {
                if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && DateTime.UtcNow - SpaceMonitor.LastNotificationHistory[db.Name] < SpaceMonitor.NotificationFrequency)
                {
                    SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is low on space but notification throttling is hit. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
                    {
                        db.Name,
                        SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? "Never" : SpaceMonitor.LastNotificationHistory[db.Name].ToString(),
                        SpaceMonitor.NotificationFrequency.TotalMinutes,
                        text3
                    });
                    return(NotificationAction.None);
                }
                Exception ex;
                if (BitlockerUtil.IsFilePathOnEncryptingVolume(db.LogFolderPath.PathName, out ex))
                {
                    SpaceMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is currently encrypting. Do not raise alert.", db.Name);
                    return(NotificationAction.None);
                }
                if (ex != null)
                {
                    string text4 = string.Format("IsFilePathOnEncryptingVolume({0}) failed: {1}", db.LogFolderPath.PathName, ex.Message);
                    ReplayCrimsonEvents.BitlockerQueryFailed.LogPeriodic <string, Exception>(Environment.MachineName, DiagCore.DefaultEventSuppressionInterval, text4, ex);
                }
                EventNotificationItem eventNotificationItem = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Critical);
                eventNotificationItem.Publish(false);
                if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name))
                {
                    SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.UtcNow;
                }
                else
                {
                    SpaceMonitor.LastNotificationHistory.Add(db.Name, DateTime.UtcNow);
                }
                return(NotificationAction.RedRaised);
            }
            else
            {
                text3 = string.Format("{0} Status is OK - EdbFreeSpace={1:0.##} GB [{2}], LogFreeSpace={3:0.##} GB [{4}], Threshold={5:0.##} GB", new object[]
                {
                    db.Name,
                    num4,
                    text,
                    num2,
                    text2,
                    num
                });
                if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && SpaceMonitor.LastNotificationHistory[db.Name] != DateTime.MinValue)
                {
                    EventNotificationItem eventNotificationItem2 = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Informational);
                    eventNotificationItem2.Publish(false);
                    SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.MinValue;
                    return(NotificationAction.GreenRaised);
                }
                SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has enough space but red notification was never raised. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
                {
                    db.Name,
                    SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? SpaceMonitor.LastNotificationHistory[db.Name].ToString() : "Never",
                    SpaceMonitor.NotificationFrequency.TotalMinutes,
                    text3
                });
                return(NotificationAction.None);
            }
        }