// Token: 0x0600194F RID: 6479 RVA: 0x0006886C File Offset: 0x00066A6C
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state)
{
LocalizedString result = base.RunPrereqs(state);
if (!result.IsEmpty)
{
return(result);
}
result = FailedSuspendedCopyAutoReseedWorkflow.hookableCheckExVolumes.Value(base.Context);
if (!result.IsEmpty)
{
return(result);
}
result = FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context);
if (!result.IsEmpty)
{
return(result);
}
if (base.Context.TargetCopyStatus.CopyStatus.ActionInitiator == ActionInitiatorType.Administrator)
{
base.TraceError("RunPrereqs(): The DatabaseCopy has been suspended by an Administrator so AutoReseed will not be attempted.", new object[0]);
return(ReplayStrings.AutoReseedFailedAdminSuspended);
}
if (base.Context.TargetCopyStatus.CopyStatus.ReseedBlocked)
{
string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage);
base.TraceError("RunPrereqs(): The DatabaseCopy is marked as ReseedBlocked so AutoReseed will not be attempted. Database copy ErrorMessage: {0}", new object[]
{
messageOrNoneString
});
return(ReplayStrings.AutoReseedFailedReseedBlocked(messageOrNoneString));
}
this.ResetWorkflowRecoveryActionIfNecessary(state);
if (!state.IsLastReseedRecoveryActionPending())
{
base.TraceDebug("RunPrereqs(): Running the workflow for the first time, so starting with the Resume action. LastReseedRecoveryAction = {0}", new object[]
{
state.LastReseedRecoveryAction
});
state.UpdateReseedRecoveryAction(ReseedState.Resume);
}
if (state.LastReseedRecoveryAction == ReseedState.Resume)
{
result = this.RunPrereqsForResume(state);
}
if (state.LastReseedRecoveryAction == ReseedState.AssignSpare)
{
result = this.RunPrereqsForAssignSpare(state);
}
if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed)
{
result = this.RunPrereqsForInPlaceReseed(state);
}
return(result);
}
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state)
{
LocalizedString result = base.RunPrereqs(state);
if (!result.IsEmpty)
{
return(result);
}
result = FailedSuspendedCopyAutoReseedWorkflow.CheckExchangeVolumesPresent(base.Context);
if (!result.IsEmpty)
{
return(result);
}
return(FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context));
}
// Token: 0x06001955 RID: 6485 RVA: 0x00068D58 File Offset: 0x00066F58
private bool DoesMountPointNeedToBeFixed(IEnumerable <CopyStatusClientCachedEntry> targetDbSet, out MountedFolderPath volumeName)
{
volumeName = MountedFolderPath.Empty;
RpcDatabaseCopyStatus2 copyStatus = base.Context.TargetCopyStatus.CopyStatus;
if (FailedSuspendedCopyAutoReseedWorkflow.DoesCopyHaveMountPointConfigured(copyStatus))
{
return(false);
}
CopyStatusClientCachedEntry copyStatusClientCachedEntry = targetDbSet.FirstOrDefault((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && FailedSuspendedCopyAutoReseedWorkflow.DoesCopyHaveMountPointConfigured(status.CopyStatus));
if (copyStatusClientCachedEntry == null)
{
base.TraceError("Database copy is missing a mount point, but no other copies in the grouping have mount points either.", new object[0]);
return(false);
}
volumeName = new MountedFolderPath(copyStatusClientCachedEntry.CopyStatus.DatabaseVolumeName);
return(true);
}
// Token: 0x06001964 RID: 6500 RVA: 0x0006A0D8 File Offset: 0x000682D8
private static void BeginSeedRpc(FailedSuspendedCopyAutoReseedWorkflow wf)
{
using (SeederClient seederClient = SeederClient.Create(wf.Context.TargetServerName.Fqdn, wf.Context.Database.Name, null, wf.Context.TargetServer.AdminDisplayVersion))
{
try
{
seederClient.CancelDbSeed(wf.Context.Database.Guid);
seederClient.EndDbSeed(wf.Context.Database.Guid);
wf.TraceDebug("Prior seed instance has been Cancelled.", new object[0]);
}
catch (SeederInstanceNotFoundException ex)
{
wf.TraceDebug("Prior seed instance cleanup failed with: {0}", new object[]
{
ex
});
}
seederClient.PrepareDbSeedAndBegin(wf.Context.Database.Guid, false, true, false, false, true, true, string.Empty, null, string.Empty, null, null, SeederRpcFlags.SkipSettingReseedAutoReseedState);
}
}
// Token: 0x06001958 RID: 6488 RVA: 0x00069248 File Offset: 0x00067448
private IEnumerable <CopyStatusClientCachedEntry> FindTargetDbSetFromNeighbors(AutoReseedContext context)
{
IEnumerable <CopyStatusClientCachedEntry> enumerable = null;
IEnumerable <CopyStatusClientCachedEntry> enumerable2 = null;
Dictionary <AmServerName, IEnumerable <CopyStatusClientCachedEntry> > copyStatusesForDag = context.CopyStatusesForDag;
IEnumerable <KeyValuePair <AmServerName, IEnumerable <CopyStatusClientCachedEntry> > > enumerable3 = from kvp in copyStatusesForDag
let server = kvp.Key
let isTargetServer = this.IsTargetServer(server, context.TargetServerName)
where this.ShouldSelectServer(isTargetServer)
orderby isTargetServer descending
select kvp;
foreach (KeyValuePair <AmServerName, IEnumerable <CopyStatusClientCachedEntry> > keyValuePair in enumerable3)
{
AmServerName key = keyValuePair.Key;
IEnumerable <CopyStatusClientCachedEntry> value = keyValuePair.Value;
IEnumerable <CopyStatusClientCachedEntry> source = from status in value
where status.Result == CopyStatusRpcResult.Success && FailedSuspendedCopyAutoReseedWorkflow.DoesCopyHaveMountPointConfigured(status.CopyStatus)
select status;
IEnumerable <IGrouping <string, CopyStatusClientCachedEntry> > databaseSets = source.GroupBy((CopyStatusClientCachedEntry status) => status.CopyStatus.DatabaseVolumeName, StringComparer.OrdinalIgnoreCase);
IGrouping <string, CopyStatusClientCachedEntry> grouping = FailedSuspendedCopyAutoReseedWorkflow.FindDatabaseGrouping(context, databaseSets);
if (grouping != null)
{
int num = grouping.Count <CopyStatusClientCachedEntry>();
if (num == context.Dag.AutoDagDatabaseCopiesPerVolume)
{
enumerable2 = grouping;
break;
}
}
}
if (enumerable2 != null)
{
Dictionary <Guid, CopyStatusClientCachedEntry> remoteDbMap = enumerable2.ToDictionary((CopyStatusClientCachedEntry status) => status.DbGuid);
enumerable = from status in context.CopyStatusesForTargetServer
where remoteDbMap.ContainsKey(status.DbGuid)
select status;
enumerable = enumerable.ToArray <CopyStatusClientCachedEntry>();
}
return(enumerable);
}
// Token: 0x060018F8 RID: 6392 RVA: 0x00066668 File Offset: 0x00064868
public void BeginAutoReseedIfNecessary(AutoReseedContext context)
{
Guid guid = context.Database.Guid;
string name = context.Database.Name;
if (context.TargetCopyStatus == null)
{
AutoReseedWorkflowLauncher.Tracer.TraceError <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' has null TargetCopyStatus. Skipping.", name, context.TargetServerName);
return;
}
if (context.TargetCopyStatus.Result != CopyStatusRpcResult.Success)
{
AutoReseedWorkflowLauncher.Tracer.TraceError((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Skipping since GetCopyStatus RPC to database copy '{0}\\{1}' failed. Result: {2}. Error: {3}.", new object[]
{
name,
context.TargetServerName,
context.TargetCopyStatus.Result,
context.TargetCopyStatus.LastException
});
return;
}
this.RunNeverMountedActiveWorkflow(context);
this.RunHealthyCopyWorkflowIfNecessary(context);
ExtendedErrorInfo extendedErrorInfo = context.TargetCopyStatus.CopyStatus.ExtendedErrorInfo;
if (context.TargetCopyStatus.CopyStatus.CopyStatus == CopyStatusEnum.Failed && (extendedErrorInfo == null || extendedErrorInfo.FailureException == null || !(extendedErrorInfo.FailureException is ReplayServiceRpcCopyStatusTimeoutException)))
{
if (context.TargetCopyStatus.IsActive)
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is 'Failed' but active. Skipping.", name, context.TargetServerName);
return;
}
if (this.m_suppression.ReportWorkflowLaunchConditionMet(AutoReseedWorkflowType.FailedCopy, guid, CatalogAutoReseedWorkflow.CatalogAutoReseedReason.None, 1))
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' has been 'Failed' for at least {2}. Launching the FailedCopyWorkflow workflow.", name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbFailedSuppresionInterval);
FailedCopyWorkflow failedCopyWorkflow = new FailedCopyWorkflow(context, context.TargetCopyStatus.CopyStatus.ErrorMessage);
failedCopyWorkflow.Execute();
return;
}
AutoReseedWorkflowLauncher.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is 'Failed' but launching the recovery workflow is being skipped due to either initial suppression of {2}, or periodic suppression of {3}.", new object[]
{
name,
context.TargetServerName,
AutoReseedWorkflowSuppression.s_dbFailedSuppresionInterval,
AutoReseedWorkflowSuppression.s_dbFailedSuppresionInterval
});
return;
}
else
{
if (context.TargetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.FailedAndSuspended)
{
if (context.TargetCopyStatus.CopyStatus.ContentIndexStatus == ContentIndexStatusType.FailedAndSuspended)
{
if (context.CopyStatusesForTargetDatabase.All((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && (status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Disabled || status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Suspended || status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.FailedAndSuspended)))
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, Guid, string>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: AutoReseed workflow launcher detected all catalogs failed for database '{0}' [{1}]: {2}.", context.Database.Name, context.Database.Guid, context.TargetCopyStatus.CopyStatus.ErrorMessage);
ReplayCrimsonEvents.AutoReseedWorkflowAllCatalogFailed.Log <string, Guid, string, string>(context.Database.Name, context.Database.Guid, "FailedSuspendedCatalogRebuildWorkflow", context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage);
if (this.m_suppression.ReportWorkflowLaunchConditionMet(AutoReseedWorkflowType.FailedSuspendedCatalogRebuild, guid, CatalogAutoReseedWorkflow.CatalogAutoReseedReason.None, 1))
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database '{0}' has no catalogs in healthy state for at least {1}. Launching the recovery workflow.", name, AutoReseedWorkflowSuppression.s_ciRebuildSuppresionInterval);
FailedSuspendedCatalogRebuildWorkflow failedSuspendedCatalogRebuildWorkflow = new FailedSuspendedCatalogRebuildWorkflow(context, context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage);
failedSuspendedCatalogRebuildWorkflow.Execute();
return;
}
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, TimeSpan, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database '{0}' has no catalogs in healthy state but launching the recovery workflow is being skipped due to either initial suppression of {1}, or periodic suppression of {2}.", name, AutoReseedWorkflowSuppression.s_ciRebuildSuppresionInterval, AutoReseedWorkflowSuppression.s_ciRebuildRetryInterval);
return;
}
}
if (!this.TryLaunchCatalogAutoReseedWorkflow(context, name))
{
this.m_suppression.ReportNoWorkflowsNeedToLaunch(guid);
}
return;
}
if (context.TargetCopyStatus.IsActive)
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is 'FailedAndSuspended' but active. Skipping.", name, context.TargetServerName);
return;
}
if (this.m_suppression.ReportWorkflowLaunchConditionMet(AutoReseedWorkflowType.FailedSuspendedCopyAutoReseed, guid, CatalogAutoReseedWorkflow.CatalogAutoReseedReason.None, 1))
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' has been FailedAndSuspended for at least {2}. Launching the recovery workflow.", name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbReseedSuppresionInterval);
FailedSuspendedCopyAutoReseedWorkflow failedSuspendedCopyAutoReseedWorkflow = new FailedSuspendedCopyAutoReseedWorkflow(context, context.TargetCopyStatus.CopyStatus.ErrorMessage);
failedSuspendedCopyAutoReseedWorkflow.Execute();
return;
}
AutoReseedWorkflowLauncher.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is FailedAndSuspended but launching the recovery workflow is being skipped due to either initial suppression of {2}, or periodic suppression of {3}.", new object[]
{
name,
context.TargetServerName,
AutoReseedWorkflowSuppression.s_dbReseedSuppresionInterval,
AutoReseedWorkflowSuppression.s_dbReseedRetryInterval
});
return;
}
}
// Token: 0x0600195E RID: 6494 RVA: 0x000696C0 File Offset: 0x000678C0
private LocalizedString RunPrereqsForAssignSpare(AutoReseedWorkflowState state)
{
this.TraceBeginPrereqs(state);
if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax)
{
int autoReseedDbFailedAssignSpareRetryCountMax = RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax;
base.TraceError("RunPrereqsForAssignSpare(): Failing 'AssignSpare' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedAssignSpareRetryCountMax ({1}). Workflow will try InPlaceReseed stage next.", new object[]
{
state.ReseedRecoveryActionRetryCount,
autoReseedDbFailedAssignSpareRetryCountMax
});
ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRetryExceeded.Log <string, Guid, string, string, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedAssignSpareRetryCountMax);
state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed);
return(ReplayStrings.AutoReseedFailedSeedRetryExceeded(autoReseedDbFailedAssignSpareRetryCountMax));
}
state.UpdateReseedRecoveryAction(ReseedState.AssignSpare);
if (string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName) || string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName))
{
return(ReplayStrings.AutoReseedFailedToFindTargetVolumeName(AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.VolumeInfoLastError)));
}
MountedFolderPath mountedFolderPath = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName);
MountedFolderPath other = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName);
if (!mountedFolderPath.Equals(other))
{
return(ReplayStrings.AutoReseedLogAndDbNotOnSameVolume);
}
IEnumerable <CopyStatusClientCachedEntry> enumerable = this.FindTargetDbSetFromDatabaseGroup(base.Context);
if (enumerable == null)
{
enumerable = this.FindTargetDbSetFromNeighbors(base.Context);
if (enumerable == null)
{
return(ReplayStrings.AutoReseedFailedToFindVolumeName);
}
}
MountedFolderPath mountedFolderPath2;
if (this.IsVolumeRecentlyAssigned(enumerable, out mountedFolderPath2))
{
ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRecentlyAssigned.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason);
state.AssignedVolumeName = mountedFolderPath2.Path;
state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed);
}
else if (this.DoesMountPointNeedToBeFixed(enumerable, out mountedFolderPath2))
{
base.TraceDebug("Database copy is missing a mount point, and it will be fixed up to point to volume '{0}'", new object[]
{
mountedFolderPath2
});
this.m_volumeForMissingMountPoint = mountedFolderPath2;
}
else
{
string databaseNames = FailedSuspendedCopyAutoReseedWorkflow.GetDatabaseNames(enumerable);
if (!enumerable.All((CopyStatusClientCachedEntry status) => !status.IsActive))
{
return(ReplayStrings.AutoReseedNotAllCopiesPassive(databaseNames));
}
if (!enumerable.All((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.CopyStatus == CopyStatusEnum.FailedAndSuspended))
{
return(ReplayStrings.AutoReseedNotAllCopiesOnVolumeFailedSuspended(databaseNames));
}
}
this.m_targetDbSet = enumerable;
return(LocalizedString.Empty);
}
internal bool DetectLowSpace(IADDatabase db, IMonitoringADConfig adConfig)
{
if (db.ReplicationType != ReplicationType.Remote)
{
return(false);
}
if (!FailedSuspendedCopyAutoReseedWorkflow.IsAutoReseedEnabled(db))
{
return(false);
}
int num = RegistryParameters.SpaceMonitorMinHealthyCount;
DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, num, this.m_statusLookup, adConfig, null, true);
IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run();
CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus;
if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.Healthy || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL)
{
return(false);
}
if (targetCopyStatus.CopyStatus.LastLogInfoIsStale || (targetCopyStatus.CopyStatus.GetCopyQueueLength() < (long)RegistryParameters.SpaceMonitorCopyQueueThreshold && targetCopyStatus.CopyStatus.GetReplayQueueLength() < (long)RegistryParameters.SpaceMonitorReplayQueueThreshold))
{
return(false);
}
if (healthValidationResult.IsTargetCopyHealthy)
{
num++;
}
long num2 = (long)(targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1048576UL);
long num3 = (long)(targetCopyStatus.CopyStatus.DiskTotalSpaceBytes / 1048576UL);
long num4 = 0L;
bool flag = false;
if (num2 <= targetCopyStatus.CopyStatus.GetCopyQueueLength() || num3 <= 0L)
{
flag = true;
}
else
{
num4 = (num2 - targetCopyStatus.CopyStatus.GetCopyQueueLength()) * 1048576L / 1048576L;
if (num4 <= (long)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB)
{
flag = true;
}
}
SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has {1}MB remaining = {2}MB effective. MinHealthy={3} ReportedHealthy={4}", new object[]
{
db.Name,
num2,
num4,
num,
healthValidationResult.HealthyCopiesCount
});
if (!flag)
{
this.m_actionSuppression.ReportSuccess(db.Guid);
return(false);
}
bool flag2 = !this.m_actionSuppression.ReportFailure(db.Guid, this.SuppressionWindow);
SpaceMonitor.Tracer.TraceError <string, bool>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Local copy of Database '{0}' is low on space. Action suppressed = {1}", db.Name, flag2);
if (flag2)
{
return(false);
}
ReplayCrimsonEvents.SpaceMonitorLowSpaceDetected.LogPeriodic <string, Guid, long, long, string, long, long, long, long, int, int>(db.Guid, this.m_lowSpaceLoggingPeriod, db.Name, db.Guid, num3, num2, string.Format("{0:0.000}", (double)num2 / (double)num3 * 100.0), targetCopyStatus.CopyStatus.GetCopyQueueLength(), targetCopyStatus.CopyStatus.GetReplayQueueLength(), targetCopyStatus.CopyStatus.LowestLogPresent, targetCopyStatus.CopyStatus.LastLogCopied, healthValidationResult.HealthyCopiesCount, num);
return(healthValidationResult.HealthyCopiesCount >= num);
}