// Token: 0x0600195D RID: 6493 RVA: 0x00069564 File Offset: 0x00067764
private LocalizedString RunPrereqsForResume(AutoReseedWorkflowState state)
{
LocalizedString result = LocalizedString.Empty;
this.TraceBeginPrereqs(state);
if (base.Context.TargetCopyStatus.CopyStatus.ResumeBlocked)
{
string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage);
result = ReplayStrings.AutoReseedFailedResumeBlocked(messageOrNoneString);
base.TraceError("RunPrereqsForResume(): The DatabaseCopy is marked as ResumeBlocked, so Resume stage is being skipped. Workflow will try AssignSpare stage next. DatabaseCopy has ErrorMessage: {0}", new object[]
{
messageOrNoneString
});
ReplayCrimsonEvents.AutoReseedWorkflowDbResumeBlocked.Log <string, Guid, string, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, messageOrNoneString);
state.UpdateReseedRecoveryAction(ReseedState.AssignSpare);
}
else if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedResumeRetryCountMax)
{
int autoReseedDbFailedResumeRetryCountMax = RegistryParameters.AutoReseedDbFailedResumeRetryCountMax;
result = ReplayStrings.AutoReseedFailedResumeRetryExceeded(autoReseedDbFailedResumeRetryCountMax);
base.TraceError("RunPrereqsForResume(): Failing 'Resume' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedResumeRetryCountMax ({1}). Workflow will try AssignSpare stage next.", new object[]
{
state.ReseedRecoveryActionRetryCount,
autoReseedDbFailedResumeRetryCountMax
});
ReplayCrimsonEvents.AutoReseedWorkflowDbResumeRetryExceeded.Log <string, Guid, string, string, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedResumeRetryCountMax);
state.UpdateReseedRecoveryAction(ReseedState.AssignSpare);
}
return(result);
}
private bool ShouldThrottleExecution(AutoReseedWorkflowState state, out Exception exception)
{
bool flag = false;
exception = null;
TimeSpan t;
if (this.GetWorkflowElapsedExecutionTime(state, out t))
{
TimeSpan throttlingInterval = this.GetThrottlingInterval(state);
if (t < throttlingInterval)
{
flag = true;
exception = new AutoReseedThrottledException(this.Context.Database.Name, this.Context.TargetServerName.NetbiosName, throttlingInterval.ToString());
}
AutoReseedWorkflow.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}]: Throttling interval ({3}). Last execution time ({4}). Throttle = {5}.", new object[]
{
this.WorkflowName,
this.Context.Database.Name,
this.Context.Database.Guid,
throttlingInterval,
state.WorkflowExecutionTime,
flag
});
}
else
{
AutoReseedWorkflow.Tracer.TraceDebug <string, string, Guid>((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}] will not be throttled because this is the first time it is being executed.", this.WorkflowName, this.Context.Database.Name, this.Context.Database.Guid);
}
return(flag);
}
// Token: 0x06001962 RID: 6498 RVA: 0x00069DC8 File Offset: 0x00067FC8
private Exception ExecuteAssignSpare(AutoReseedWorkflowState state)
{
Exception ex = null;
try
{
bool flag = true;
MountedFolderPath mountedFolderPath;
if (!MountedFolderPath.IsNullOrEmpty(this.m_volumeForMissingMountPoint))
{
this.LogBeginExecute(state);
flag = false;
string name = base.Context.Database.Name;
DatabaseSpareInfo dbInfo = new DatabaseSpareInfo(name, new MountedFolderPath(Path.Combine(base.Context.Dag.AutoDagDatabasesRootFolderPath.PathName, name)));
ExchangeVolume exchangeVolume = base.Context.VolumeManager.FixupMountPointForDatabase(dbInfo, this.m_volumeForMissingMountPoint);
this.UpdateVolumeInfoCopyState(base.Context.Database.Guid);
ReplayCrimsonEvents.AutoReseedWorkflowDbMountPointMissing.Log <string, Guid, string, string, MountedFolderPath, MountedFolderPath>(name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, exchangeVolume.ExchangeVolumeMountPoint, exchangeVolume.VolumeName);
}
else if (this.IsVolumeRecentlyAssigned(this.m_targetDbSet, out mountedFolderPath))
{
flag = false;
base.TraceDebug("Skipping assigning a new volume since a volume was recently assigned. ReseedRecoveryActionRetryCount: {0}", new object[]
{
state.ReseedRecoveryActionRetryCount
});
state.AssignedVolumeName = mountedFolderPath.Path;
}
if (flag)
{
this.LogBeginExecute(state);
DatabaseSpareInfo[] dbInfos = (from status in this.m_targetDbSet
select new DatabaseSpareInfo(status.CopyStatus.DBName, new MountedFolderPath(Path.Combine(base.Context.Dag.AutoDagDatabasesRootFolderPath.PathName, status.CopyStatus.DBName)))).ToArray <DatabaseSpareInfo>();
ExchangeVolume exchangeVolume2 = base.Context.VolumeManager.AssignSpare(dbInfos);
base.TraceDebug("Assigned spare volume: {0}", new object[]
{
exchangeVolume2.VolumeName.Path
});
foreach (CopyStatusClientCachedEntry copyStatusClientCachedEntry in this.m_targetDbSet)
{
this.UpdateVolumeInfoCopyState(copyStatusClientCachedEntry.DbGuid);
}
ReplayCrimsonEvents.AutoReseedWorkflowDbFailedAssignSpareSucceeded.Log <string, Guid, string, string, MountedFolderPath, MountedFolderPath>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, exchangeVolume2.ExchangeVolumeMountPoint, exchangeVolume2.VolumeName);
state.AssignedVolumeName = exchangeVolume2.VolumeName.Path;
}
else
{
base.TraceDebug("Re-using previously assigned volume: {0}", new object[]
{
state.AssignedVolumeName
});
}
state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed);
}
catch (DatabaseVolumeInfoException ex2)
{
ex = ex2;
}
this.LogExecuteCompleted(state, ex);
return(ex);
}
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state)
{
LocalizedString result = base.RunPrereqs(state);
if (!result.IsEmpty)
{
return(result);
}
int num = base.Context.CopyStatusesForTargetDatabase.Count((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Healthy);
if (num == 0)
{
AutoReseedWorkflow.Tracer.TraceDebug <string, Guid, string>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow detected all catalogs failed for database '{0}' [{1}]: {2}.", base.Context.Database.Name, base.Context.Database.Guid, base.Context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage);
ReplayCrimsonEvents.AutoReseedWorkflowAllCatalogFailed.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.Context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage);
return(ReplayStrings.AutoReseedAllCatalogFailed(base.Context.Database.Name));
}
if (num == 1)
{
AutoReseedWorkflow.Tracer.TraceDebug <string, Guid>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow detected only one catalog copy is healthy for database '{0}' [{1}].", base.Context.Database.Name, base.Context.Database.Guid);
ReplayCrimsonEvents.AutoReseedWorkflowSingleCatalogHealthy.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.Context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage);
}
int num2;
if (!base.Context.ReseedLimiter.TryStartCiSeed(out num2))
{
base.TraceError("CatalogAutoReseedWorkflow is being skipped for now because maximum number of concurrent seeds has been reached: {0}", new object[]
{
num2
});
return(ReplayStrings.AutoReseedTooManyConcurrentSeeds(num2));
}
return(LocalizedString.Empty);
}
protected override Exception ExecuteInternal(AutoReseedWorkflowState state)
{
RpcDatabaseCopyStatus2 copyStatus = base.Context.TargetCopyStatus.CopyStatus;
TimeSpan timeSpan = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedPeriodicIntervalInSecs);
base.TraceDebug("Calling SuspendAndFailLocalDatabaseCopy() ...", new object[0]);
return(DatabaseTasks.SuspendAndFailLocalDatabaseCopy(base.Context.Database, ReplayStrings.AutoReseedFailedCopyWorkflowSuspendedCopy(timeSpan.ToString()), copyStatus.ErrorMessage, copyStatus.ErrorEventId, copyStatus.ResumeBlocked, copyStatus.ReseedBlocked, copyStatus.InPlaceReseedBlocked));
}
// Token: 0x0600194F RID: 6479 RVA: 0x0006886C File Offset: 0x00066A6C
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state)
{
LocalizedString result = base.RunPrereqs(state);
if (!result.IsEmpty)
{
return(result);
}
result = FailedSuspendedCopyAutoReseedWorkflow.hookableCheckExVolumes.Value(base.Context);
if (!result.IsEmpty)
{
return(result);
}
result = FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context);
if (!result.IsEmpty)
{
return(result);
}
if (base.Context.TargetCopyStatus.CopyStatus.ActionInitiator == ActionInitiatorType.Administrator)
{
base.TraceError("RunPrereqs(): The DatabaseCopy has been suspended by an Administrator so AutoReseed will not be attempted.", new object[0]);
return(ReplayStrings.AutoReseedFailedAdminSuspended);
}
if (base.Context.TargetCopyStatus.CopyStatus.ReseedBlocked)
{
string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage);
base.TraceError("RunPrereqs(): The DatabaseCopy is marked as ReseedBlocked so AutoReseed will not be attempted. Database copy ErrorMessage: {0}", new object[]
{
messageOrNoneString
});
return(ReplayStrings.AutoReseedFailedReseedBlocked(messageOrNoneString));
}
this.ResetWorkflowRecoveryActionIfNecessary(state);
if (!state.IsLastReseedRecoveryActionPending())
{
base.TraceDebug("RunPrereqs(): Running the workflow for the first time, so starting with the Resume action. LastReseedRecoveryAction = {0}", new object[]
{
state.LastReseedRecoveryAction
});
state.UpdateReseedRecoveryAction(ReseedState.Resume);
}
if (state.LastReseedRecoveryAction == ReseedState.Resume)
{
result = this.RunPrereqsForResume(state);
}
if (state.LastReseedRecoveryAction == ReseedState.AssignSpare)
{
result = this.RunPrereqsForAssignSpare(state);
}
if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed)
{
result = this.RunPrereqsForInPlaceReseed(state);
}
return(result);
}
// Token: 0x06001966 RID: 6502 RVA: 0x0006A1F4 File Offset: 0x000683F4
private void TraceBeginPrereqs(AutoReseedWorkflowState state)
{
if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace))
{
base.TraceDebug("Starting prereqs for '{0}' stage. Attempt number: {1}", new object[]
{
state.LastReseedRecoveryAction,
state.ReseedRecoveryActionRetryCount + 1
});
}
}
// Token: 0x06001967 RID: 6503 RVA: 0x0006A24C File Offset: 0x0006844C
private void LogBeginExecute(AutoReseedWorkflowState state)
{
if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace))
{
base.TraceDebug("Starting Execute for '{0}' stage. Attempt number: {1}", new object[]
{
state.LastReseedRecoveryAction,
state.ReseedRecoveryActionRetryCount
});
}
ReplayCrimsonEvents.AutoReseedWorkflowDbFailedExecuteStage.Log <string, Guid, string, string, ReseedState, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, state.LastReseedRecoveryAction, state.ReseedRecoveryActionRetryCount);
}
private bool ArePrereqsSatisfied(AutoReseedWorkflowState state, out Exception exception)
{
exception = null;
LocalizedString value = this.RunPrereqs(state);
if (!value.IsEmpty)
{
exception = new AutoReseedPrereqFailedException(this.Context.Database.Name, this.Context.TargetServerName.NetbiosName, value);
return(false);
}
return(true);
}
protected override Exception ExecuteInternal(AutoReseedWorkflowState state)
{
if (!base.Context.TargetCopyStatus.IsActive)
{
AutoReseedWorkflow.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "Database copy '{0}\\{1}' is not active. Switch over before rebuilding catalog.", base.Context.Database.Name, base.Context.TargetServerName);
Exception ex = FailedSuspendedCatalogRebuildWorkflow.hookableMove.Value(this);
if (ex != null)
{
return(ex);
}
}
return(FailedSuspendedCatalogRebuildWorkflow.hookableRebuildRpc.Value(this));
}
protected override Exception ExecuteInternal(AutoReseedWorkflowState state)
{
int num = int.MaxValue;
bool skipBehindCatalog = false;
if (this.catalogReseedReason == CatalogAutoReseedWorkflow.CatalogAutoReseedReason.BehindBacklog || this.catalogReseedReason == CatalogAutoReseedWorkflow.CatalogAutoReseedReason.BehindRetry)
{
num = this.WeighCiCopyStatus(base.Context.TargetCopyStatus, false);
skipBehindCatalog = true;
}
foreach (CopyStatusClientCachedEntry copyStatusClientCachedEntry in base.Context.CopyStatusesForTargetDatabase)
{
if (!copyStatusClientCachedEntry.ServerContacted.Equals(base.Context.TargetServerName) && copyStatusClientCachedEntry.Result == CopyStatusRpcResult.Success && copyStatusClientCachedEntry.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Healthy && (copyStatusClientCachedEntry.CopyStatus.CopyStatus == CopyStatusEnum.Mounted || copyStatusClientCachedEntry.CopyStatus.CopyStatus == CopyStatusEnum.Healthy || copyStatusClientCachedEntry.CopyStatus.CopyStatus == CopyStatusEnum.DisconnectedAndHealthy))
{
if (this.catalogReseedReason == CatalogAutoReseedWorkflow.CatalogAutoReseedReason.Upgrade)
{
if (copyStatusClientCachedEntry.CopyStatus.ContentIndexVersion == null)
{
continue;
}
int value = copyStatusClientCachedEntry.CopyStatus.ContentIndexVersion.Value;
VersionInfo latest = VersionInfo.Latest;
if (value != latest.QueryVersion)
{
continue;
}
}
int num2 = this.WeighCiCopyStatus(copyStatusClientCachedEntry, skipBehindCatalog);
if (num2 < num)
{
this.sourceName = copyStatusClientCachedEntry.ServerContacted.Fqdn;
num = num2;
}
}
}
AutoReseedWorkflow.Tracer.TraceDebug <string, string, AmServerName>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow: Selected '{0}' as source server for content index of database copy '{1}\\{2}'.", this.sourceName, base.Context.Database.Name, base.Context.TargetServerName);
if (string.IsNullOrEmpty(this.sourceName))
{
return(new AutoReseedCatalogSourceException(base.Context.Database.Name, base.Context.TargetServerName.NetbiosName));
}
if (base.Context.TargetCopyStatus.IsActive)
{
AutoReseedWorkflow.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow: Database copy '{0}\\{1}' is active. Fail over.", base.Context.Database.Name, base.Context.TargetServerName);
new DatabaseFailureItem(FailureNameSpace.ContentIndex, FailureTag.CatalogReseed, base.Context.Database.Guid)
{
InstanceName = base.Context.Database.Name
}.Publish();
return(new AutoReseedCatalogActiveException(base.Context.Database.Name, base.Context.TargetServerName.NetbiosName));
}
return(CatalogAutoReseedWorkflow.hookableReseedAction.Value(this));
}
protected bool GetWorkflowElapsedExecutionTime(AutoReseedWorkflowState state, out TimeSpan elapsedExecutionTime)
{
elapsedExecutionTime = TimeSpan.Zero;
if (state.WorkflowExecutionTime.Equals(DateTime.MinValue))
{
this.TraceDebug("GetWorkflowElapsedExecutionTime(): Returning 'false' since WorkflowExecutionTime is DateTime.MinValue.", new object[0]);
return(false);
}
elapsedExecutionTime = DateTime.UtcNow.Subtract(state.WorkflowExecutionTime);
this.TraceDebug("GetWorkflowElapsedExecutionTime(): Returning elapsedExecutionTime = '{0}'", new object[]
{
elapsedExecutionTime
});
return(true);
}
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state)
{
LocalizedString result = base.RunPrereqs(state);
if (!result.IsEmpty)
{
return(result);
}
if (base.Context.CopyStatusesForTargetDatabase.Any((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.ActivationPreference < base.Context.TargetCopyStatus.CopyStatus.ActivationPreference))
{
AutoReseedWorkflow.Tracer.TraceDebug <string, Guid, string>((long)this.GetHashCode(), "AutoReseed workflow launcher detected failed catalog for database '{0}' [{1}]: {2}. However, the catalog doesn't qualify to be rebuilt.", base.Context.Database.Name, base.Context.Database.Guid, base.Context.TargetCopyStatus.CopyStatus.ErrorMessage);
return(ReplayStrings.AutoReseedCatalogSkipRebuild(base.Context.Database.Name, base.Context.TargetServerName.Fqdn));
}
return(LocalizedString.Empty);
}
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state)
{
LocalizedString result = base.RunPrereqs(state);
if (!result.IsEmpty)
{
return(result);
}
result = FailedSuspendedCopyAutoReseedWorkflow.CheckExchangeVolumesPresent(base.Context);
if (!result.IsEmpty)
{
return(result);
}
return(FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context));
}
public Exception Execute()
{
Exception ex = null;
if (this.IsDisabled)
{
AutoReseedWorkflow.Tracer.TraceDebug <string, string, Guid>((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}] will not be executed because it has been disabled via regkey.", this.WorkflowName, this.Context.Database.Name, this.Context.Database.Guid);
return(null);
}
try
{
AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(this.Context.Database.Guid, this.WorkflowType);
if (this.ShouldSkipWorkflowExecution(autoReseedWorkflowState))
{
return(null);
}
this.LogWorkflowStarted();
if (this.ArePrereqsSatisfied(autoReseedWorkflowState, out ex) && !this.ShouldThrottleExecution(autoReseedWorkflowState, out ex))
{
bool flag = true;
try
{
ex = this.ExecuteInternal(autoReseedWorkflowState);
flag = false;
}
catch (RegistryParameterException ex2)
{
flag = false;
ex = new AutoReseedException(ex2.Message, ex2);
}
finally
{
if (flag)
{
ex = new AutoReseedUnhandledException(this.Context.Database.Name, this.Context.TargetServerName.NetbiosName);
}
autoReseedWorkflowState.WriteWorkflowExecutionState(ex);
}
}
}
catch (RegistryParameterException ex3)
{
ex = new AutoReseedException(ex3.Message, ex3);
}
this.LogWorkflowEnded(ex);
return(ex);
}
// Token: 0x0600194D RID: 6477 RVA: 0x00068814 File Offset: 0x00066A14
protected override TimeSpan GetThrottlingInterval(AutoReseedWorkflowState state)
{
switch (state.LastReseedRecoveryAction)
{
case ReseedState.Unknown:
case ReseedState.Resume:
case ReseedState.Completed:
return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedThrottlingIntervalInSecs_Resume));
case ReseedState.AssignSpare:
case ReseedState.InPlaceReseed:
return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedThrottlingIntervalInSecs_Reseed));
default:
return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedThrottlingIntervalInSecs_Resume));
}
}
// Token: 0x06001950 RID: 6480 RVA: 0x000689B0 File Offset: 0x00066BB0
protected override Exception ExecuteInternal(AutoReseedWorkflowState state)
{
Exception result = null;
if (state.LastReseedRecoveryAction == ReseedState.Unknown || state.LastReseedRecoveryAction == ReseedState.Resume)
{
result = this.ExecuteResume(state);
}
else if (state.LastReseedRecoveryAction == ReseedState.AssignSpare)
{
result = this.ExecuteAssignSpare(state);
}
else if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed)
{
result = this.ExecuteInPlaceReseed(state);
}
return(result);
}
// Token: 0x0600191D RID: 6429 RVA: 0x000675E8 File Offset: 0x000657E8
public static Exception TriggerInPlaceReseed(Guid dbGuid, string dbName)
{
Exception result = null;
try
{
AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(dbGuid, AutoReseedWorkflowType.FailedSuspendedCopyAutoReseed);
autoReseedWorkflowState.ResetReseedRecoveryAction();
autoReseedWorkflowState.IgnoreInPlaceOverwriteDelay = true;
autoReseedWorkflowState.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed);
ReplayCrimsonEvents.AutoReseedTriggerInPlaceReseed.LogPeriodic <string, Guid>(dbGuid, DiagCore.DefaultEventSuppressionInterval, dbName, dbGuid);
}
catch (RegistryParameterException ex)
{
result = ex;
}
return(result);
}
private bool ShouldSkipWorkflowExecution(AutoReseedWorkflowState state)
{
bool result = false;
DateTime workflowNextExecutionTime = state.WorkflowNextExecutionTime;
if (workflowNextExecutionTime != DateTime.MinValue && DateTime.UtcNow < workflowNextExecutionTime)
{
AutoReseedWorkflow.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}]: Skipping execution because WorkflowNextExecutionTime is set to '{3}'.", new object[]
{
this.WorkflowName,
this.Context.Database.Name,
this.Context.Database.Guid,
workflowNextExecutionTime
});
result = true;
}
return(result);
}
// Token: 0x06001953 RID: 6483 RVA: 0x00068B44 File Offset: 0x00066D44
private void ResetWorkflowRecoveryActionIfNecessary(AutoReseedWorkflowState state)
{
TimeSpan timeSpan;
if (!base.GetWorkflowElapsedExecutionTime(state, out timeSpan))
{
base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Doing nothing.", new object[0]);
return;
}
TimeSpan t = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedWorkflowResetIntervalInSecs);
if (!(timeSpan >= t))
{
base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Doing nothing since elapsedExecutionTime of '{0}' is too recent.", new object[]
{
timeSpan
});
return;
}
if (!state.IsLastReseedRecoveryActionPending())
{
base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Successfully completed workflow is being reset after duration of '{0}'.", new object[]
{
timeSpan
});
state.ResetReseedRecoveryAction();
return;
}
if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed && state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedReseedRetryCountMax)
{
base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Workflow is being reset after duration of '{0}'.", new object[]
{
timeSpan
});
ReplayCrimsonEvents.AutoReseedWorkflowReset.Log <string, Guid, string, string, DateTime>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, state.WorkflowExecutionTime);
state.ResetReseedRecoveryAction();
return;
}
base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Doing nothing since the workflow is still attempting to fix the copy.", new object[0]);
}
protected override Exception ExecuteInternal(AutoReseedWorkflowState state)
{
Exception ex = null;
bool flag = false;
IVolumeManager volumeManager = base.Context.VolumeManager;
IADDatabase database = base.Context.Database;
MountedFolderPath databaseMountedFolderPath = VolumeManager.GetDatabaseMountedFolderPath(base.Context.Dag.AutoDagDatabasesRootFolderPath, database.Name);
if (MountPointUtil.IsDirectoryAccessibleMountPoint(databaseMountedFolderPath.Path, out ex))
{
flag = true;
}
else
{
ReplayCrimsonEvents.AutoReseedNeverMountedActiveMissingVolume.Log <string>(database.Name);
if (volumeManager.FixActiveDatabaseMountPoint(database, base.Context.Databases, base.Context.AdConfig, out ex, true))
{
flag = true;
volumeManager.UpdateVolumeInfoCopyState(database.Guid, base.Context.ReplicaInstanceManager);
ReplayCrimsonEvents.AutoReseedNeverMountedActiveAllocatedVolume.Log <string, string>(database.Name, base.Context.TargetCopyStatus.CopyStatus.LogVolumeName);
}
else
{
AutoReseedWorkflow.Tracer.TraceError <string, string>((long)this.GetHashCode(), "DiskReclaimer: UpdateVolumeForNeverMountedActives() failed to fix up active database: '{0}' mountpoint. Error: {1}", database.Name, AmExceptionHelper.GetExceptionMessageOrNoneString(ex));
ReplayCrimsonEvents.AutoReseedFixActiveMountPointError.Log <string, string>(database.Name, AmExceptionHelper.GetExceptionMessageOrNoneString(ex));
}
}
if (!flag)
{
AutoReseedWorkflow.Tracer.TraceError <string, string>((long)this.GetHashCode(), "DiskReclaimer: UpdateVolumeForNeverMountedActives() active database: '{0}' does not have a mountpoint. Skip issuing Store Mount. Error: {1}", database.Name, AmExceptionHelper.GetExceptionMessageOrNoneString(ex));
return(ex);
}
ex = AmRpcClientHelper.AdminMountDatabaseWrapper(database);
if (ex != null)
{
AutoReseedWorkflow.Tracer.TraceError <string, string>((long)this.GetHashCode(), "DiskReclaimer: UpdateVolumeForNeverMountedActives() failed to mount active database: '{0}' mountpoint. Error: {1}", database.Name, ex.Message);
ReplayCrimsonEvents.AutoReseedMountActiveDatabaseError.Log <string, string>(database.Name, ex.Message);
}
return(ex);
}
// Token: 0x06001963 RID: 6499 RVA: 0x0006A044 File Offset: 0x00068244
private Exception ExecuteInPlaceReseed(AutoReseedWorkflowState state)
{
Exception ex = null;
try
{
state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed);
this.LogBeginExecute(state);
FailedSuspendedCopyAutoReseedWorkflow.hookableSeedRpc.Value(this);
ReplayCrimsonEvents.AutoReseedWorkflowDbFailedBeginReseedSucceeded.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason);
}
catch (SeederServerException ex2)
{
ex = ex2;
}
catch (SeederServerTransientException ex3)
{
ex = ex3;
}
this.LogExecuteCompleted(state, ex);
return(ex);
}
// Token: 0x06001961 RID: 6497 RVA: 0x00069D10 File Offset: 0x00067F10
private Exception ExecuteResume(AutoReseedWorkflowState state)
{
Exception ex = null;
try
{
state.UpdateReseedRecoveryAction(ReseedState.Resume);
this.LogBeginExecute(state);
DatabaseCopyActionFlags arg = DatabaseCopyActionFlags.Replication | DatabaseCopyActionFlags.Activation | DatabaseCopyActionFlags.SkipSettingResumeAutoReseedState;
FailedSuspendedCopyAutoReseedWorkflow.hookableResumeRpc.Value(AmServerName.LocalComputerName.Fqdn, base.Context.Database.Guid, (uint)arg);
}
catch (TaskServerException ex2)
{
ex = ex2;
}
catch (TaskServerTransientException ex3)
{
ex = ex3;
}
this.LogExecuteCompleted(state, ex);
return(ex);
}
// Token: 0x060018FA RID: 6394 RVA: 0x0006700C File Offset: 0x0006520C
private void RunHealthyCopyWorkflowIfNecessary(AutoReseedContext context)
{
Guid guid = context.Database.Guid;
string name = context.Database.Name;
bool flag = false;
if (context.TargetCopyStatus.CopyStatus.CopyStatus == CopyStatusEnum.Healthy)
{
try
{
AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(guid, AutoReseedWorkflowType.FailedSuspendedCopyAutoReseed);
AutoReseedWorkflowState autoReseedWorkflowState2 = new AutoReseedWorkflowState(guid, AutoReseedWorkflowType.ManualReseed);
AutoReseedWorkflowState autoReseedWorkflowState3 = new AutoReseedWorkflowState(guid, AutoReseedWorkflowType.ManualResume);
if (autoReseedWorkflowState2.IsLastReseedRecoveryActionPending() || autoReseedWorkflowState3.IsLastReseedRecoveryActionPending() || autoReseedWorkflowState.IsLastReseedRecoveryActionPending())
{
flag = true;
if (this.m_suppression.ReportHealthyWorkflowLaunchConditionMet(guid))
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is Healthy with some prior recovery action having been completed. Launching the HealthyCopyCompletedSeedWorkflow workflow.", name, context.TargetServerName);
HealthyCopyCompletedSeedWorkflow healthyCopyCompletedSeedWorkflow = new HealthyCopyCompletedSeedWorkflow(context);
healthyCopyCompletedSeedWorkflow.Execute();
}
else
{
AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is Healthy with some prior recovery action having been completed but launching the workflow is being skipped due suppression of {2}.", name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbHealthySuppressionInterval);
}
}
}
catch (RegistryParameterException arg)
{
AutoReseedWorkflowLauncher.Tracer.TraceError <string, AmServerName, RegistryParameterException>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Couldn't launch/execute HealthyCopyCompletedSeedWorkflow for database copy '{0}\\{1}' to potentially reset the AutoReseed state. Error: {2}", name, context.TargetServerName, arg);
}
}
if (!flag)
{
this.m_suppression.ReportHealthyWorkflowNotNeeded(guid);
}
}
// Token: 0x0600191C RID: 6428 RVA: 0x0006759C File Offset: 0x0006579C
public static Exception WriteManualWorkflowExecutionState(Guid dbGuid, AutoReseedWorkflowType manualAction)
{
Exception result = null;
try
{
AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(dbGuid, manualAction);
if (manualAction == AutoReseedWorkflowType.ManualResume)
{
autoReseedWorkflowState.LastReseedRecoveryAction = ReseedState.Resume;
}
else
{
autoReseedWorkflowState.LastReseedRecoveryAction = ReseedState.InPlaceReseed;
}
autoReseedWorkflowState.WriteWorkflowExecutionState(null);
}
catch (RegistryParameterException ex)
{
result = ex;
}
return(result);
}
// Token: 0x06001968 RID: 6504 RVA: 0x0006A2E4 File Offset: 0x000684E4
private void LogExecuteCompleted(AutoReseedWorkflowState state, Exception exception)
{
if (exception == null)
{
if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace))
{
base.TraceDebug("Execution of stage '{0}' succeeded. Attempt number: {1}", new object[]
{
state.LastReseedRecoveryAction,
state.ReseedRecoveryActionRetryCount
});
}
return;
}
if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.ErrorTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.ErrorTrace))
{
base.TraceError("Execution of stage '{0}' failed with exception: {1}", new object[]
{
state.LastReseedRecoveryAction,
exception
});
}
ReplayCrimsonEvents.AutoReseedWorkflowDbFailedExecutionStageFailed.Log <string, Guid, string, string, ReseedState, int, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, state.LastReseedRecoveryAction, state.ReseedRecoveryActionRetryCount, exception.Message);
}
// Token: 0x06001960 RID: 6496 RVA: 0x00069970 File Offset: 0x00067B70
private LocalizedString RunPrereqsForInPlaceReseed(AutoReseedWorkflowState state)
{
this.TraceBeginPrereqs(state);
int autoReseedDbFailedReseedRetryCountMax = RegistryParameters.AutoReseedDbFailedReseedRetryCountMax;
TimeSpan timeSpan = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedWorkflowResetIntervalInSecs);
timeSpan = this.GetWorkflowSuppressionInterval(timeSpan);
if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedReseedRetryCountMax)
{
state.WriteWorkflowNextExecutionDueTime(timeSpan);
base.TraceError("RunPrereqsForInPlaceReseed(): Failing 'InPlaceReseed' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedReseedRetryCountMax ({1}). Workflow will next run after {2}.", new object[]
{
state.ReseedRecoveryActionRetryCount,
autoReseedDbFailedReseedRetryCountMax,
timeSpan
});
ReplayCrimsonEvents.AutoReseedWorkflowDbReseedRetryExceeded.Log <string, Guid, string, string, int, TimeSpan>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedReseedRetryCountMax, timeSpan);
return(ReplayStrings.AutoReseedFailedSeedRetryExceeded(autoReseedDbFailedReseedRetryCountMax));
}
if (base.Context.TargetCopyStatus.CopyStatus.InPlaceReseedBlocked)
{
IEnumerable <CopyStatusClientCachedEntry> enumerable = this.FindTargetDbSetFromDatabaseGroup(base.Context);
if (enumerable == null)
{
enumerable = this.FindTargetDbSetFromNeighbors(base.Context);
if (enumerable == null)
{
return(ReplayStrings.AutoReseedFailedToFindVolumeName);
}
}
MountedFolderPath mountedFolderPath;
if (!this.IsVolumeRecentlyAssigned(enumerable, out mountedFolderPath))
{
string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage);
state.WriteWorkflowNextExecutionDueTime(timeSpan);
state.ReseedRecoveryActionRetryCount = RegistryParameters.AutoReseedDbFailedReseedRetryCountMax;
base.TraceError("RunPrereqsForInPlaceReseed(): Failing 'InPlaceReseed' prereqs since in-place reseed is blocked. DatabaseCopy has ErrorMessage: {0}. Workflow will next run after {1}.", new object[]
{
messageOrNoneString,
timeSpan
});
ReplayCrimsonEvents.AutoReseedWorkflowDbInPlaceReseedBlocked.Log <string, Guid, string, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, messageOrNoneString);
return(ReplayStrings.AutoReseedFailedInPlaceReseedBlocked(messageOrNoneString));
}
}
string edbFilePath = base.Context.Database.EdbFilePath.PathName;
if (File.Exists(edbFilePath) && !state.IgnoreInPlaceOverwriteDelay)
{
DateTime lastWriteTimeUtc = DateTime.MaxValue;
Exception ex = MountPointUtil.HandleIOExceptions(delegate
{
FileInfo fileInfo = new FileInfo(edbFilePath);
lastWriteTimeUtc = fileInfo.LastWriteTimeUtc;
});
if (ex != null)
{
lastWriteTimeUtc = base.Context.TargetCopyStatus.CopyStatus.LastStatusTransitionTime;
base.TraceError("RunPrereqsForInPlaceReseed(): Failed to read LastWriteTimeUtc of edb file '{0}'. Using LastStatusTransitionTime of '{1}' instead. Exception: {2}", new object[]
{
edbFilePath,
lastWriteTimeUtc,
ex
});
}
TimeSpan timeSpan2 = DateTime.UtcNow.Subtract(lastWriteTimeUtc);
TimeSpan timeSpan3 = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedInPlaceReseedDelayInSecs);
base.TraceDebug("RunPrereqsForInPlaceReseed(): Found EDB file at '{0}'. The copy was last FailedAndSuspended '{1}' duration ago. Configured reseed delay is: '{2}'.", new object[]
{
edbFilePath,
timeSpan2,
timeSpan3
});
if (timeSpan2 <= timeSpan3)
{
TimeSpan workflowSuppressionInterval = this.GetWorkflowSuppressionInterval(timeSpan3);
state.WriteWorkflowNextExecutionDueTime(workflowSuppressionInterval);
base.TraceError("RunPrereqsForInPlaceReseed(): EDB file exists and Reseed is being attempted too soon. Reseed will be blocked until the configured reseed delay period. Workflow will next run after {0}.", new object[]
{
workflowSuppressionInterval
});
ReplayCrimsonEvents.AutoReseedWorkflowInPlaceReseedTooSoon.Log <string, Guid, string, string, string, TimeSpan, TimeSpan, TimeSpan>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, edbFilePath, timeSpan2, timeSpan3, workflowSuppressionInterval);
return(ReplayStrings.AutoReseedInPlaceReseedTooSoon(timeSpan2.ToString(), timeSpan3.ToString()));
}
}
int num;
if (!base.Context.ReseedLimiter.TryStartSeed(out num))
{
base.TraceError("RunPrereqsForInPlaceReseed(): Seed is being skipped for now because maximum number of concurrent seeds has been reached: {0}", new object[]
{
num
});
return(ReplayStrings.AutoReseedTooManyConcurrentSeeds(num));
}
return(LocalizedString.Empty);
}
protected override TimeSpan GetThrottlingInterval(AutoReseedWorkflowState state)
{
return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedCiThrottlingIntervalInSecs));
}
// Token: 0x0600195E RID: 6494 RVA: 0x000696C0 File Offset: 0x000678C0
private LocalizedString RunPrereqsForAssignSpare(AutoReseedWorkflowState state)
{
this.TraceBeginPrereqs(state);
if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax)
{
int autoReseedDbFailedAssignSpareRetryCountMax = RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax;
base.TraceError("RunPrereqsForAssignSpare(): Failing 'AssignSpare' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedAssignSpareRetryCountMax ({1}). Workflow will try InPlaceReseed stage next.", new object[]
{
state.ReseedRecoveryActionRetryCount,
autoReseedDbFailedAssignSpareRetryCountMax
});
ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRetryExceeded.Log <string, Guid, string, string, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedAssignSpareRetryCountMax);
state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed);
return(ReplayStrings.AutoReseedFailedSeedRetryExceeded(autoReseedDbFailedAssignSpareRetryCountMax));
}
state.UpdateReseedRecoveryAction(ReseedState.AssignSpare);
if (string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName) || string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName))
{
return(ReplayStrings.AutoReseedFailedToFindTargetVolumeName(AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.VolumeInfoLastError)));
}
MountedFolderPath mountedFolderPath = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName);
MountedFolderPath other = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName);
if (!mountedFolderPath.Equals(other))
{
return(ReplayStrings.AutoReseedLogAndDbNotOnSameVolume);
}
IEnumerable <CopyStatusClientCachedEntry> enumerable = this.FindTargetDbSetFromDatabaseGroup(base.Context);
if (enumerable == null)
{
enumerable = this.FindTargetDbSetFromNeighbors(base.Context);
if (enumerable == null)
{
return(ReplayStrings.AutoReseedFailedToFindVolumeName);
}
}
MountedFolderPath mountedFolderPath2;
if (this.IsVolumeRecentlyAssigned(enumerable, out mountedFolderPath2))
{
ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRecentlyAssigned.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason);
state.AssignedVolumeName = mountedFolderPath2.Path;
state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed);
}
else if (this.DoesMountPointNeedToBeFixed(enumerable, out mountedFolderPath2))
{
base.TraceDebug("Database copy is missing a mount point, and it will be fixed up to point to volume '{0}'", new object[]
{
mountedFolderPath2
});
this.m_volumeForMissingMountPoint = mountedFolderPath2;
}
else
{
string databaseNames = FailedSuspendedCopyAutoReseedWorkflow.GetDatabaseNames(enumerable);
if (!enumerable.All((CopyStatusClientCachedEntry status) => !status.IsActive))
{
return(ReplayStrings.AutoReseedNotAllCopiesPassive(databaseNames));
}
if (!enumerable.All((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.CopyStatus == CopyStatusEnum.FailedAndSuspended))
{
return(ReplayStrings.AutoReseedNotAllCopiesOnVolumeFailedSuspended(databaseNames));
}
}
this.m_targetDbSet = enumerable;
return(LocalizedString.Empty);
}
protected override TimeSpan GetThrottlingInterval(AutoReseedWorkflowState state)
{
return(TimeSpan.Zero);
}