private void ProcessDatabaseInternal(IADDatabase db, IMonitoringADConfig adConfig)
{
if (db.ReplicationType != ReplicationType.Remote)
{
ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Database '{0}' is not a replicated database. Skipping this database.", db.Name);
return;
}
IADDatabaseCopy databaseCopy = this.GetDatabaseCopy(db, AmServerName.LocalComputerName);
EnhancedTimeSpan replayLagTime = databaseCopy.ReplayLagTime;
if (replayLagTime == EnhancedTimeSpan.Zero)
{
ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' is not a lag copy. Skipping this database.", db.Name);
return;
}
ReplayLagManager.Tracer.TraceDebug <string, EnhancedTimeSpan>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' is a lag copy. Configured lag: {1}. Processing this database.", db.Name, replayLagTime);
DatabaseAvailabilityValidator databaseAvailabilityValidator = new DatabaseAvailabilityValidator(db, ReplayLagManager.NUM_AVAILABLE_NONLAG_COPIES_MIN, this.m_statusLookup, adConfig, null, true);
IHealthValidationResult healthValidationResult = databaseAvailabilityValidator.Run();
int num = ReplayLagManager.NUM_AVAILABLE_NONLAG_COPIES_MIN;
if (healthValidationResult.IsTargetCopyHealthy)
{
num++;
ReplayLagManager.Tracer.TraceDebug <string, int>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local lag-copy of Database '{0}' is available. Increasing the minimum available copy count to: {1}", db.Name, num);
}
CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus;
if (healthValidationResult.HealthyCopiesCount >= num)
{
ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' has met availability critera. Replay Lag will be re-instated as necessary.", db.Name);
if (this.m_errorSuppression.ReportSuccess(db.Guid, ReplayLagManager.EnableLagSuppressionWindow))
{
this.EnableReplayLag(db, targetCopyStatus);
return;
}
ReplayLagManager.Tracer.TraceDebug <string, double>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: '{0}': Skipping EnableReplayLag() due to transient suppression of {1} secs", db.Name, ReplayLagManager.EnableLagSuppressionWindow.TotalSeconds);
return;
}
else
{
ReplayLagManager.Tracer.TraceDebug <string>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: Local copy of Database '{0}' has *NOT* met availability critera. Replay Lag will be played down as necessary.", db.Name);
string errorMessageWithoutFullStatus = healthValidationResult.ErrorMessageWithoutFullStatus;
if (this.m_errorSuppression.ReportFailure(db.Guid, ReplayLagManager.DisableLagSuppressionWindow))
{
this.DisableReplayLag(db, targetCopyStatus, errorMessageWithoutFullStatus);
return;
}
ReplayLagManager.Tracer.TraceDebug <string, double>((long)this.GetHashCode(), "ReplayLagManager.ProcessDatabase: '{0}': Skipping DisableReplayLag() due to transient suppression of {1} secs", db.Name, ReplayLagManager.DisableLagSuppressionWindow.TotalSeconds);
ReplayCrimsonEvents.RLMDisableReplayLagRequestSuppressed.LogPeriodic <string, string, Guid, string, TimeSpan>(Environment.MachineName, DateTimeHelper.FourHours, db.Name, Environment.MachineName, db.Guid, errorMessageWithoutFullStatus, ReplayLagManager.DisableLagSuppressionWindow);
return;
}
}
private void RaiseDatabaseAvailabilityAlertIfNecessary(IADDatabase db, IMonitoringADConfig adConfig)
{
DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseAvailabilityAlertIfNecessary: DB '{0}': Starting DB-level availability checks.", db.Name);
DatabaseAvailabilityValidator databaseAvailabilityValidator = new DatabaseAvailabilityValidator(db, DatabaseHealthMonitor.NUM_HEALTHY_COPIES_MIN, this.m_statusLookup, adConfig, this.m_propertyUpdateTracker, false);
IHealthValidationResult result = databaseAvailabilityValidator.Run();
this.m_dbAlerts.OneAvailableCopy.RaiseAppropriateAlertIfNecessary(result);
if (adConfig.Dag.DatacenterActivationMode == DatacenterActivationModeOption.DagOnly)
{
this.m_dbAlerts.OneAvailableCopySite.RaiseAppropriateAlertIfNecessary(result);
return;
}
DatabaseHealthMonitor.Tracer.TraceDebug <string, DatacenterActivationModeOption>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseAvailabilityAlertIfNecessary: DB '{0}': Skipping raising site alerts because DatacenterActivationMode is '{1}'.", db.Name, adConfig.Dag.DatacenterActivationMode);
}
private static string GetActiveServerName(IHealthValidationResultMinimal validationResult)
{
IHealthValidationResult healthValidationResult = (IHealthValidationResult)validationResult;
string result = null;
if (healthValidationResult.ActiveCopyStatus != null)
{
result = healthValidationResult.ActiveCopyStatus.ServerContacted.NetbiosName;
}
else if (healthValidationResult.TargetCopyStatus != null && healthValidationResult.TargetCopyStatus.ActiveServer != null)
{
result = healthValidationResult.TargetCopyStatus.ActiveServer.NetbiosName;
}
return(result);
}
private void RaiseDatabasePotentialRedundancyAlertIfNecessary(IADDatabase db, IMonitoringADConfig adConfig)
{
if (this.stopWatch.Elapsed.TotalSeconds < (double)RegistryParameters.DatabaseHealthCheckDelayInRaisingDatabasePotentialRedundancyAlertDueToServiceStartInSec)
{
DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabasePotentialRedundancyAlertIfNecessary: Service just started, skip DB-level checks.", db.Name);
return;
}
DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabasePotentialRedundancyAlertIfNecessary: DB '{0}': Starting DB-level checks.", db.Name);
DatabaseReplicationNotStalledValidator databaseReplicationNotStalledValidator = new DatabaseReplicationNotStalledValidator(db, this.m_statusLookup, adConfig, this.m_propertyUpdateTracker, false);
IHealthValidationResult healthValidationResult = databaseReplicationNotStalledValidator.Run();
int databaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin = RegistryParameters.DatabaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin;
if (!healthValidationResult.IsValidationSuccessful && healthValidationResult.TotalPassiveCopiesCount < databaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin)
{
DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabasePotentialRedundancyAlertIfNecessary: DB '{0}': Skipping raising potential redundancy alert because not enough passive copies are found", db.Name);
string text = string.Format("{0}-NotEnoughTotalPassiveCopies", healthValidationResult.Identity);
ReplayCrimsonEvents.DatabaseLevelPotentialRedundancyCheckFailedButNotEnoughTotalPassiveCopies.LogPeriodic <string, int, int, int, int, string, Guid>(text, TimeSpan.FromMinutes(60.0), healthValidationResult.Identity, healthValidationResult.HealthyCopiesCount, healthValidationResult.HealthyPassiveCopiesCount, healthValidationResult.TotalPassiveCopiesCount, databaseHealthCheckPotentialOneCopyTotalPassiveCopiesRequiredMin, "", healthValidationResult.IdentityGuid);
return;
}
this.m_dbAlerts.PotentialOneRedundantCopy.RaiseAppropriateAlertIfNecessary(healthValidationResult);
}
private void RunLogic(bool isFastLagPlaydownDesired, long replayQ)
{
if (RegistryParameters.DisableDatabaseScan)
{
this.isEnabled = false;
return;
}
if (isFastLagPlaydownDesired)
{
if (this.DisableScan())
{
this.LogDisable("FastLagPlaydownDesired");
}
return;
}
if (!this.isEnabled && ExDateTime.UtcNow - this.lastDisableTime < this.Parameters.MinDisabledWindow)
{
return;
}
if (this.isEnabled && replayQ < this.Parameters.MinReplayQOfInterest)
{
return;
}
if (this.isLagCopy)
{
if (replayQ > this.Parameters.MaxReplayQForBehindCopy)
{
if (this.DisableScan())
{
this.LogDisable(string.Format("Lag too behind. ReplayQ={0} MaxQ={1}", replayQ, this.Parameters.MaxReplayQForBehindCopy));
return;
}
}
else if (!this.isEnabled && replayQ < this.Parameters.MaxReplayQForAvailableCopy && this.EnableScan())
{
this.LogEnable(string.Format("Lag catching up. ReplayQ={0} MaxQ={1}", replayQ, this.Parameters.MaxReplayQForAvailableCopy));
}
return;
}
bool flag = this.sensor.DatabaseHasSufficientAvailability();
if (this.sensor.FailedToRead)
{
if (replayQ > this.Parameters.MaxReplayQForAvailableCopy && this.DisableScan())
{
this.LogDisable("DatabaseHasSufficientAvailability failed");
}
return;
}
IHealthValidationResult lastReading = this.sensor.LastReading;
if (!flag)
{
if (this.DisableScan())
{
string reason = string.Format("Database '{0}' has insufficient availability. MinAvail={1} CurAvail={2}", this.Database.Name, this.sensor.MinAvailableCopies, lastReading.HealthyCopiesCount);
this.LogDisable(reason);
}
return;
}
if (this.isEnabled)
{
if (replayQ <= this.Parameters.MaxReplayQForAvailableCopy)
{
return;
}
if (lastReading.IsTargetCopyHealthy)
{
if (this.DisableScan())
{
this.LogDisable(string.Format("Availability at risk. ReplayQ={0}", replayQ));
}
return;
}
if (replayQ > this.Parameters.MaxReplayQForBehindCopy || (this.keepingUp != null && this.keepingUp.ReplayQueueNotKeepingUp > 0L))
{
if (this.DisableScan())
{
this.LogDisable(string.Format("Too far behind. ReplayQ={0}", replayQ));
}
return;
}
}
else if (lastReading.IsTargetCopyHealthy)
{
if (replayQ > this.Parameters.MinReplayQOfInterest)
{
return;
}
if (this.EnableScan())
{
this.LogEnable(string.Format("Available copy recovered. ReplayQ={0}", replayQ));
}
return;
}
else if (this.EnableScan())
{
this.LogEnable(string.Format("Availability is ok due to other copies. This copy can resume scanning. ReplayQ={0}", replayQ));
}
}
internal bool DetectLowSpace(IADDatabase db, IMonitoringADConfig adConfig)
{
if (db.ReplicationType != ReplicationType.Remote)
{
return(false);
}
if (!FailedSuspendedCopyAutoReseedWorkflow.IsAutoReseedEnabled(db))
{
return(false);
}
int num = RegistryParameters.SpaceMonitorMinHealthyCount;
DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, num, this.m_statusLookup, adConfig, null, true);
IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run();
CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus;
if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.Healthy || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
{
return(false);
}
if (targetCopyStatus.CopyStatus.LastLogInfoIsStale || (targetCopyStatus.CopyStatus.GetCopyQueueLength() < (long)RegistryParameters.SpaceMonitorCopyQueueThreshold && targetCopyStatus.CopyStatus.GetReplayQueueLength() < (long)RegistryParameters.SpaceMonitorReplayQueueThreshold))
{
return(false);
}
if (healthValidationResult.IsTargetCopyHealthy)
{
num++;
}
long num2 = (long)(targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1048576UL);
long num3 = (long)(targetCopyStatus.CopyStatus.DiskTotalSpaceBytes / 1048576UL);
long num4 = 0L;
bool flag = false;
if (num2 <= targetCopyStatus.CopyStatus.GetCopyQueueLength() || num3 <= 0L)
{
flag = true;
}
else
{
num4 = (num2 - targetCopyStatus.CopyStatus.GetCopyQueueLength()) * 1048576L / 1048576L;
if (num4 <= (long)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB)
{
flag = true;
}
}
SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has {1}MB remaining = {2}MB effective. MinHealthy={3} ReportedHealthy={4}", new object[]
{
db.Name,
num2,
num4,
num,
healthValidationResult.HealthyCopiesCount
});
if (!flag)
{
this.m_actionSuppression.ReportSuccess(db.Guid);
return(false);
}
bool flag2 = !this.m_actionSuppression.ReportFailure(db.Guid, this.SuppressionWindow);
SpaceMonitor.Tracer.TraceError <string, bool>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Local copy of Database '{0}' is low on space. Action suppressed = {1}", db.Name, flag2);
if (flag2)
{
return(false);
}
ReplayCrimsonEvents.SpaceMonitorLowSpaceDetected.LogPeriodic <string, Guid, long, long, string, long, long, long, long, int, int>(db.Guid, this.m_lowSpaceLoggingPeriod, db.Name, db.Guid, num3, num2, string.Format("{0:0.000}", (double)num2 / (double)num3 * 100.0), targetCopyStatus.CopyStatus.GetCopyQueueLength(), targetCopyStatus.CopyStatus.GetReplayQueueLength(), targetCopyStatus.CopyStatus.LowestLogPresent, targetCopyStatus.CopyStatus.LastLogCopied, healthValidationResult.HealthyCopiesCount, num);
return(healthValidationResult.HealthyCopiesCount >= num);
}
internal NotificationAction NotifyLowDiskSpace(IADDatabase db, IMonitoringADConfig adConfig)
{
DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, 1, this.m_statusLookup, adConfig, null, true);
IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run();
CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus;
if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
{
return(NotificationAction.None);
}
double num = (double)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB / 1024.0;
double num2 = targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1024.0 / 1024.0 / 1024.0;
double num3 = 0.0;
double num4 = 0.0;
string text = SpaceMonitor.FindVolume(new DirectoryInfo(db.EdbFilePath.PathName));
string text2 = SpaceMonitor.FindVolume(new DirectoryInfo(db.LogFolderPath.PathName));
if (!string.IsNullOrEmpty(text) && !string.IsNullOrEmpty(text2) && !text.Equals(text2, StringComparison.OrdinalIgnoreCase) && db.EdbFilePath.IsLocalFull)
{
ulong num5 = 0UL;
ulong num6 = 0UL;
DiskHelper.GetFreeSpace(db.EdbFilePath.PathName, out num5, out num6);
num3 = num5 / 1024UL / 1024UL / 1024UL;
num4 = num6 / 1024UL / 1024UL / 1024UL;
}
bool flag = false;
string text3 = string.Empty;
if (num2 <= num)
{
text3 = string.Format("'{0}' is low on log volume space [{1}]. Current={2:0.##} GB, Threshold={3:0.##} GB", new object[]
{
db.Name,
text2,
num2,
num
});
flag = true;
}
if (num3 != 0.0 && num4 <= num)
{
text3 += string.Format("{0}'{1}' is low on EDB volume space [{2}]. Current={3:0.##} GB, Threshold={4:0.##} GB", new object[]
{
Environment.NewLine,
db.Name,
text,
num4,
num
});
flag = true;
}
if (flag)
{
if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && DateTime.UtcNow - SpaceMonitor.LastNotificationHistory[db.Name] < SpaceMonitor.NotificationFrequency)
{
SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is low on space but notification throttling is hit. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
{
db.Name,
SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? "Never" : SpaceMonitor.LastNotificationHistory[db.Name].ToString(),
SpaceMonitor.NotificationFrequency.TotalMinutes,
text3
});
return(NotificationAction.None);
}
Exception ex;
if (BitlockerUtil.IsFilePathOnEncryptingVolume(db.LogFolderPath.PathName, out ex))
{
SpaceMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is currently encrypting. Do not raise alert.", db.Name);
return(NotificationAction.None);
}
if (ex != null)
{
string text4 = string.Format("IsFilePathOnEncryptingVolume({0}) failed: {1}", db.LogFolderPath.PathName, ex.Message);
ReplayCrimsonEvents.BitlockerQueryFailed.LogPeriodic <string, Exception>(Environment.MachineName, DiagCore.DefaultEventSuppressionInterval, text4, ex);
}
EventNotificationItem eventNotificationItem = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Critical);
eventNotificationItem.Publish(false);
if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name))
{
SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.UtcNow;
}
else
{
SpaceMonitor.LastNotificationHistory.Add(db.Name, DateTime.UtcNow);
}
return(NotificationAction.RedRaised);
}
else
{
text3 = string.Format("{0} Status is OK - EdbFreeSpace={1:0.##} GB [{2}], LogFreeSpace={3:0.##} GB [{4}], Threshold={5:0.##} GB", new object[]
{
db.Name,
num4,
text,
num2,
text2,
num
});
if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && SpaceMonitor.LastNotificationHistory[db.Name] != DateTime.MinValue)
{
EventNotificationItem eventNotificationItem2 = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Informational);
eventNotificationItem2.Publish(false);
SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.MinValue;
return(NotificationAction.GreenRaised);
}
SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has enough space but red notification was never raised. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
{
db.Name,
SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? SpaceMonitor.LastNotificationHistory[db.Name].ToString() : "Never",
SpaceMonitor.NotificationFrequency.TotalMinutes,
text3
});
return(NotificationAction.None);
}
}
