private void RaiseDatabaseRedundancyAlertIfNecessary(IADDatabase db, IMonitoringADConfig adConfig)
{
DatabaseHealthMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseRedundancyAlertIfNecessary: DB '{0}': Starting DB-level redundancy checks.", db.Name);
DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, DatabaseHealthMonitor.NUM_HEALTHY_COPIES_MIN, this.m_statusLookup, adConfig, this.m_propertyUpdateTracker, false);
IHealthValidationResult result = databaseRedundancyValidator.Run();
this.m_dbAlerts.OneRedundantCopy.RaiseAppropriateAlertIfNecessary(result);
this.m_dbAlerts.TwoCopy.RaiseAppropriateAlertIfNecessary(result);
this.m_dbAlerts.StaleStatus.RaiseAppropriateAlertIfNecessary(result);
if (adConfig.Dag.DatacenterActivationMode == DatacenterActivationModeOption.DagOnly)
{
this.m_dbAlerts.OneRedundantCopySite.RaiseAppropriateAlertIfNecessary(result);
return;
}
DatabaseHealthMonitor.Tracer.TraceDebug <string, DatacenterActivationModeOption>((long)this.GetHashCode(), "DatabaseHealthMonitor.RaiseDatabaseRedundancyAlertIfNecessary: DB '{0}': Skipping raising site alerts because DatacenterActivationMode is '{1}'.", db.Name, adConfig.Dag.DatacenterActivationMode);
}
internal bool DetectLowSpace(IADDatabase db, IMonitoringADConfig adConfig)
{
if (db.ReplicationType != ReplicationType.Remote)
{
return(false);
}
if (!FailedSuspendedCopyAutoReseedWorkflow.IsAutoReseedEnabled(db))
{
return(false);
}
int num = RegistryParameters.SpaceMonitorMinHealthyCount;
DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, num, this.m_statusLookup, adConfig, null, true);
IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run();
CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus;
if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.Healthy || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
{
return(false);
}
if (targetCopyStatus.CopyStatus.LastLogInfoIsStale || (targetCopyStatus.CopyStatus.GetCopyQueueLength() < (long)RegistryParameters.SpaceMonitorCopyQueueThreshold && targetCopyStatus.CopyStatus.GetReplayQueueLength() < (long)RegistryParameters.SpaceMonitorReplayQueueThreshold))
{
return(false);
}
if (healthValidationResult.IsTargetCopyHealthy)
{
num++;
}
long num2 = (long)(targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1048576UL);
long num3 = (long)(targetCopyStatus.CopyStatus.DiskTotalSpaceBytes / 1048576UL);
long num4 = 0L;
bool flag = false;
if (num2 <= targetCopyStatus.CopyStatus.GetCopyQueueLength() || num3 <= 0L)
{
flag = true;
}
else
{
num4 = (num2 - targetCopyStatus.CopyStatus.GetCopyQueueLength()) * 1048576L / 1048576L;
if (num4 <= (long)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB)
{
flag = true;
}
}
SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has {1}MB remaining = {2}MB effective. MinHealthy={3} ReportedHealthy={4}", new object[]
{
db.Name,
num2,
num4,
num,
healthValidationResult.HealthyCopiesCount
});
if (!flag)
{
this.m_actionSuppression.ReportSuccess(db.Guid);
return(false);
}
bool flag2 = !this.m_actionSuppression.ReportFailure(db.Guid, this.SuppressionWindow);
SpaceMonitor.Tracer.TraceError <string, bool>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Local copy of Database '{0}' is low on space. Action suppressed = {1}", db.Name, flag2);
if (flag2)
{
return(false);
}
ReplayCrimsonEvents.SpaceMonitorLowSpaceDetected.LogPeriodic <string, Guid, long, long, string, long, long, long, long, int, int>(db.Guid, this.m_lowSpaceLoggingPeriod, db.Name, db.Guid, num3, num2, string.Format("{0:0.000}", (double)num2 / (double)num3 * 100.0), targetCopyStatus.CopyStatus.GetCopyQueueLength(), targetCopyStatus.CopyStatus.GetReplayQueueLength(), targetCopyStatus.CopyStatus.LowestLogPresent, targetCopyStatus.CopyStatus.LastLogCopied, healthValidationResult.HealthyCopiesCount, num);
return(healthValidationResult.HealthyCopiesCount >= num);
}
internal NotificationAction NotifyLowDiskSpace(IADDatabase db, IMonitoringADConfig adConfig)
{
DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, 1, this.m_statusLookup, adConfig, null, true);
IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run();
CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus;
if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
{
return(NotificationAction.None);
}
double num = (double)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB / 1024.0;
double num2 = targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1024.0 / 1024.0 / 1024.0;
double num3 = 0.0;
double num4 = 0.0;
string text = SpaceMonitor.FindVolume(new DirectoryInfo(db.EdbFilePath.PathName));
string text2 = SpaceMonitor.FindVolume(new DirectoryInfo(db.LogFolderPath.PathName));
if (!string.IsNullOrEmpty(text) && !string.IsNullOrEmpty(text2) && !text.Equals(text2, StringComparison.OrdinalIgnoreCase) && db.EdbFilePath.IsLocalFull)
{
ulong num5 = 0UL;
ulong num6 = 0UL;
DiskHelper.GetFreeSpace(db.EdbFilePath.PathName, out num5, out num6);
num3 = num5 / 1024UL / 1024UL / 1024UL;
num4 = num6 / 1024UL / 1024UL / 1024UL;
}
bool flag = false;
string text3 = string.Empty;
if (num2 <= num)
{
text3 = string.Format("'{0}' is low on log volume space [{1}]. Current={2:0.##} GB, Threshold={3:0.##} GB", new object[]
{
db.Name,
text2,
num2,
num
});
flag = true;
}
if (num3 != 0.0 && num4 <= num)
{
text3 += string.Format("{0}'{1}' is low on EDB volume space [{2}]. Current={3:0.##} GB, Threshold={4:0.##} GB", new object[]
{
Environment.NewLine,
db.Name,
text,
num4,
num
});
flag = true;
}
if (flag)
{
if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && DateTime.UtcNow - SpaceMonitor.LastNotificationHistory[db.Name] < SpaceMonitor.NotificationFrequency)
{
SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is low on space but notification throttling is hit. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
{
db.Name,
SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? "Never" : SpaceMonitor.LastNotificationHistory[db.Name].ToString(),
SpaceMonitor.NotificationFrequency.TotalMinutes,
text3
});
return(NotificationAction.None);
}
Exception ex;
if (BitlockerUtil.IsFilePathOnEncryptingVolume(db.LogFolderPath.PathName, out ex))
{
SpaceMonitor.Tracer.TraceDebug <string>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' is currently encrypting. Do not raise alert.", db.Name);
return(NotificationAction.None);
}
if (ex != null)
{
string text4 = string.Format("IsFilePathOnEncryptingVolume({0}) failed: {1}", db.LogFolderPath.PathName, ex.Message);
ReplayCrimsonEvents.BitlockerQueryFailed.LogPeriodic <string, Exception>(Environment.MachineName, DiagCore.DefaultEventSuppressionInterval, text4, ex);
}
EventNotificationItem eventNotificationItem = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Critical);
eventNotificationItem.Publish(false);
if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name))
{
SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.UtcNow;
}
else
{
SpaceMonitor.LastNotificationHistory.Add(db.Name, DateTime.UtcNow);
}
return(NotificationAction.RedRaised);
}
else
{
text3 = string.Format("{0} Status is OK - EdbFreeSpace={1:0.##} GB [{2}], LogFreeSpace={3:0.##} GB [{4}], Threshold={5:0.##} GB", new object[]
{
db.Name,
num4,
text,
num2,
text2,
num
});
if (SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) && SpaceMonitor.LastNotificationHistory[db.Name] != DateTime.MinValue)
{
EventNotificationItem eventNotificationItem2 = new EventNotificationItem("MSExchangeDagMgmt", "EdbAndLogVolSpace", db.Name.ToUpper(), text3, text3, ResultSeverityLevel.Informational);
eventNotificationItem2.Publish(false);
SpaceMonitor.LastNotificationHistory[db.Name] = DateTime.MinValue;
return(NotificationAction.GreenRaised);
}
SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has enough space but red notification was never raised. LastNotify={1}, Throttling={2}mins, Message={3}", new object[]
{
db.Name,
SpaceMonitor.LastNotificationHistory.ContainsKey(db.Name) ? SpaceMonitor.LastNotificationHistory[db.Name].ToString() : "Never",
SpaceMonitor.NotificationFrequency.TotalMinutes,
text3
});
return(NotificationAction.None);
}
}
