// Token: 0x0600195D RID: 6493 RVA: 0x00069564 File Offset: 0x00067764 private LocalizedString RunPrereqsForResume(AutoReseedWorkflowState state) { LocalizedString result = LocalizedString.Empty; this.TraceBeginPrereqs(state); if (base.Context.TargetCopyStatus.CopyStatus.ResumeBlocked) { string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage); result = ReplayStrings.AutoReseedFailedResumeBlocked(messageOrNoneString); base.TraceError("RunPrereqsForResume(): The DatabaseCopy is marked as ResumeBlocked, so Resume stage is being skipped. Workflow will try AssignSpare stage next. DatabaseCopy has ErrorMessage: {0}", new object[] { messageOrNoneString }); ReplayCrimsonEvents.AutoReseedWorkflowDbResumeBlocked.Log <string, Guid, string, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, messageOrNoneString); state.UpdateReseedRecoveryAction(ReseedState.AssignSpare); } else if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedResumeRetryCountMax) { int autoReseedDbFailedResumeRetryCountMax = RegistryParameters.AutoReseedDbFailedResumeRetryCountMax; result = ReplayStrings.AutoReseedFailedResumeRetryExceeded(autoReseedDbFailedResumeRetryCountMax); base.TraceError("RunPrereqsForResume(): Failing 'Resume' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedResumeRetryCountMax ({1}). Workflow will try AssignSpare stage next.", new object[] { state.ReseedRecoveryActionRetryCount, autoReseedDbFailedResumeRetryCountMax }); ReplayCrimsonEvents.AutoReseedWorkflowDbResumeRetryExceeded.Log <string, Guid, string, string, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedResumeRetryCountMax); state.UpdateReseedRecoveryAction(ReseedState.AssignSpare); } return(result); }
private bool ShouldThrottleExecution(AutoReseedWorkflowState state, out Exception exception) { bool flag = false; exception = null; TimeSpan t; if (this.GetWorkflowElapsedExecutionTime(state, out t)) { TimeSpan throttlingInterval = this.GetThrottlingInterval(state); if (t < throttlingInterval) { flag = true; exception = new AutoReseedThrottledException(this.Context.Database.Name, this.Context.TargetServerName.NetbiosName, throttlingInterval.ToString()); } AutoReseedWorkflow.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}]: Throttling interval ({3}). Last execution time ({4}). Throttle = {5}.", new object[] { this.WorkflowName, this.Context.Database.Name, this.Context.Database.Guid, throttlingInterval, state.WorkflowExecutionTime, flag }); } else { AutoReseedWorkflow.Tracer.TraceDebug <string, string, Guid>((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}] will not be throttled because this is the first time it is being executed.", this.WorkflowName, this.Context.Database.Name, this.Context.Database.Guid); } return(flag); }
// Token: 0x06001962 RID: 6498 RVA: 0x00069DC8 File Offset: 0x00067FC8 private Exception ExecuteAssignSpare(AutoReseedWorkflowState state) { Exception ex = null; try { bool flag = true; MountedFolderPath mountedFolderPath; if (!MountedFolderPath.IsNullOrEmpty(this.m_volumeForMissingMountPoint)) { this.LogBeginExecute(state); flag = false; string name = base.Context.Database.Name; DatabaseSpareInfo dbInfo = new DatabaseSpareInfo(name, new MountedFolderPath(Path.Combine(base.Context.Dag.AutoDagDatabasesRootFolderPath.PathName, name))); ExchangeVolume exchangeVolume = base.Context.VolumeManager.FixupMountPointForDatabase(dbInfo, this.m_volumeForMissingMountPoint); this.UpdateVolumeInfoCopyState(base.Context.Database.Guid); ReplayCrimsonEvents.AutoReseedWorkflowDbMountPointMissing.Log <string, Guid, string, string, MountedFolderPath, MountedFolderPath>(name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, exchangeVolume.ExchangeVolumeMountPoint, exchangeVolume.VolumeName); } else if (this.IsVolumeRecentlyAssigned(this.m_targetDbSet, out mountedFolderPath)) { flag = false; base.TraceDebug("Skipping assigning a new volume since a volume was recently assigned. ReseedRecoveryActionRetryCount: {0}", new object[] { state.ReseedRecoveryActionRetryCount }); state.AssignedVolumeName = mountedFolderPath.Path; } if (flag) { this.LogBeginExecute(state); DatabaseSpareInfo[] dbInfos = (from status in this.m_targetDbSet select new DatabaseSpareInfo(status.CopyStatus.DBName, new MountedFolderPath(Path.Combine(base.Context.Dag.AutoDagDatabasesRootFolderPath.PathName, status.CopyStatus.DBName)))).ToArray <DatabaseSpareInfo>(); ExchangeVolume exchangeVolume2 = base.Context.VolumeManager.AssignSpare(dbInfos); base.TraceDebug("Assigned spare volume: {0}", new object[] { exchangeVolume2.VolumeName.Path }); foreach (CopyStatusClientCachedEntry copyStatusClientCachedEntry in this.m_targetDbSet) { this.UpdateVolumeInfoCopyState(copyStatusClientCachedEntry.DbGuid); } ReplayCrimsonEvents.AutoReseedWorkflowDbFailedAssignSpareSucceeded.Log <string, Guid, string, string, MountedFolderPath, MountedFolderPath>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, exchangeVolume2.ExchangeVolumeMountPoint, exchangeVolume2.VolumeName); state.AssignedVolumeName = exchangeVolume2.VolumeName.Path; } else { base.TraceDebug("Re-using previously assigned volume: {0}", new object[] { state.AssignedVolumeName }); } state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed); } catch (DatabaseVolumeInfoException ex2) { ex = ex2; } this.LogExecuteCompleted(state, ex); return(ex); }
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state) { LocalizedString result = base.RunPrereqs(state); if (!result.IsEmpty) { return(result); } int num = base.Context.CopyStatusesForTargetDatabase.Count((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Healthy); if (num == 0) { AutoReseedWorkflow.Tracer.TraceDebug <string, Guid, string>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow detected all catalogs failed for database '{0}' [{1}]: {2}.", base.Context.Database.Name, base.Context.Database.Guid, base.Context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage); ReplayCrimsonEvents.AutoReseedWorkflowAllCatalogFailed.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.Context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage); return(ReplayStrings.AutoReseedAllCatalogFailed(base.Context.Database.Name)); } if (num == 1) { AutoReseedWorkflow.Tracer.TraceDebug <string, Guid>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow detected only one catalog copy is healthy for database '{0}' [{1}].", base.Context.Database.Name, base.Context.Database.Guid); ReplayCrimsonEvents.AutoReseedWorkflowSingleCatalogHealthy.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.Context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage); } int num2; if (!base.Context.ReseedLimiter.TryStartCiSeed(out num2)) { base.TraceError("CatalogAutoReseedWorkflow is being skipped for now because maximum number of concurrent seeds has been reached: {0}", new object[] { num2 }); return(ReplayStrings.AutoReseedTooManyConcurrentSeeds(num2)); } return(LocalizedString.Empty); }
protected override Exception ExecuteInternal(AutoReseedWorkflowState state) { RpcDatabaseCopyStatus2 copyStatus = base.Context.TargetCopyStatus.CopyStatus; TimeSpan timeSpan = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedPeriodicIntervalInSecs); base.TraceDebug("Calling SuspendAndFailLocalDatabaseCopy() ...", new object[0]); return(DatabaseTasks.SuspendAndFailLocalDatabaseCopy(base.Context.Database, ReplayStrings.AutoReseedFailedCopyWorkflowSuspendedCopy(timeSpan.ToString()), copyStatus.ErrorMessage, copyStatus.ErrorEventId, copyStatus.ResumeBlocked, copyStatus.ReseedBlocked, copyStatus.InPlaceReseedBlocked)); }
// Token: 0x0600194F RID: 6479 RVA: 0x0006886C File Offset: 0x00066A6C protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state) { LocalizedString result = base.RunPrereqs(state); if (!result.IsEmpty) { return(result); } result = FailedSuspendedCopyAutoReseedWorkflow.hookableCheckExVolumes.Value(base.Context); if (!result.IsEmpty) { return(result); } result = FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context); if (!result.IsEmpty) { return(result); } if (base.Context.TargetCopyStatus.CopyStatus.ActionInitiator == ActionInitiatorType.Administrator) { base.TraceError("RunPrereqs(): The DatabaseCopy has been suspended by an Administrator so AutoReseed will not be attempted.", new object[0]); return(ReplayStrings.AutoReseedFailedAdminSuspended); } if (base.Context.TargetCopyStatus.CopyStatus.ReseedBlocked) { string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage); base.TraceError("RunPrereqs(): The DatabaseCopy is marked as ReseedBlocked so AutoReseed will not be attempted. Database copy ErrorMessage: {0}", new object[] { messageOrNoneString }); return(ReplayStrings.AutoReseedFailedReseedBlocked(messageOrNoneString)); } this.ResetWorkflowRecoveryActionIfNecessary(state); if (!state.IsLastReseedRecoveryActionPending()) { base.TraceDebug("RunPrereqs(): Running the workflow for the first time, so starting with the Resume action. LastReseedRecoveryAction = {0}", new object[] { state.LastReseedRecoveryAction }); state.UpdateReseedRecoveryAction(ReseedState.Resume); } if (state.LastReseedRecoveryAction == ReseedState.Resume) { result = this.RunPrereqsForResume(state); } if (state.LastReseedRecoveryAction == ReseedState.AssignSpare) { result = this.RunPrereqsForAssignSpare(state); } if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed) { result = this.RunPrereqsForInPlaceReseed(state); } return(result); }
// Token: 0x06001966 RID: 6502 RVA: 0x0006A1F4 File Offset: 0x000683F4 private void TraceBeginPrereqs(AutoReseedWorkflowState state) { if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace)) { base.TraceDebug("Starting prereqs for '{0}' stage. Attempt number: {1}", new object[] { state.LastReseedRecoveryAction, state.ReseedRecoveryActionRetryCount + 1 }); } }
// Token: 0x06001967 RID: 6503 RVA: 0x0006A24C File Offset: 0x0006844C private void LogBeginExecute(AutoReseedWorkflowState state) { if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace)) { base.TraceDebug("Starting Execute for '{0}' stage. Attempt number: {1}", new object[] { state.LastReseedRecoveryAction, state.ReseedRecoveryActionRetryCount }); } ReplayCrimsonEvents.AutoReseedWorkflowDbFailedExecuteStage.Log <string, Guid, string, string, ReseedState, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, state.LastReseedRecoveryAction, state.ReseedRecoveryActionRetryCount); }
private bool ArePrereqsSatisfied(AutoReseedWorkflowState state, out Exception exception) { exception = null; LocalizedString value = this.RunPrereqs(state); if (!value.IsEmpty) { exception = new AutoReseedPrereqFailedException(this.Context.Database.Name, this.Context.TargetServerName.NetbiosName, value); return(false); } return(true); }
protected override Exception ExecuteInternal(AutoReseedWorkflowState state) { if (!base.Context.TargetCopyStatus.IsActive) { AutoReseedWorkflow.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "Database copy '{0}\\{1}' is not active. Switch over before rebuilding catalog.", base.Context.Database.Name, base.Context.TargetServerName); Exception ex = FailedSuspendedCatalogRebuildWorkflow.hookableMove.Value(this); if (ex != null) { return(ex); } } return(FailedSuspendedCatalogRebuildWorkflow.hookableRebuildRpc.Value(this)); }
protected override Exception ExecuteInternal(AutoReseedWorkflowState state) { int num = int.MaxValue; bool skipBehindCatalog = false; if (this.catalogReseedReason == CatalogAutoReseedWorkflow.CatalogAutoReseedReason.BehindBacklog || this.catalogReseedReason == CatalogAutoReseedWorkflow.CatalogAutoReseedReason.BehindRetry) { num = this.WeighCiCopyStatus(base.Context.TargetCopyStatus, false); skipBehindCatalog = true; } foreach (CopyStatusClientCachedEntry copyStatusClientCachedEntry in base.Context.CopyStatusesForTargetDatabase) { if (!copyStatusClientCachedEntry.ServerContacted.Equals(base.Context.TargetServerName) && copyStatusClientCachedEntry.Result == CopyStatusRpcResult.Success && copyStatusClientCachedEntry.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Healthy && (copyStatusClientCachedEntry.CopyStatus.CopyStatus == CopyStatusEnum.Mounted || copyStatusClientCachedEntry.CopyStatus.CopyStatus == CopyStatusEnum.Healthy || copyStatusClientCachedEntry.CopyStatus.CopyStatus == CopyStatusEnum.DisconnectedAndHealthy)) { if (this.catalogReseedReason == CatalogAutoReseedWorkflow.CatalogAutoReseedReason.Upgrade) { if (copyStatusClientCachedEntry.CopyStatus.ContentIndexVersion == null) { continue; } int value = copyStatusClientCachedEntry.CopyStatus.ContentIndexVersion.Value; VersionInfo latest = VersionInfo.Latest; if (value != latest.QueryVersion) { continue; } } int num2 = this.WeighCiCopyStatus(copyStatusClientCachedEntry, skipBehindCatalog); if (num2 < num) { this.sourceName = copyStatusClientCachedEntry.ServerContacted.Fqdn; num = num2; } } } AutoReseedWorkflow.Tracer.TraceDebug <string, string, AmServerName>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow: Selected '{0}' as source server for content index of database copy '{1}\\{2}'.", this.sourceName, base.Context.Database.Name, base.Context.TargetServerName); if (string.IsNullOrEmpty(this.sourceName)) { return(new AutoReseedCatalogSourceException(base.Context.Database.Name, base.Context.TargetServerName.NetbiosName)); } if (base.Context.TargetCopyStatus.IsActive) { AutoReseedWorkflow.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "CatalogAutoReseedWorkflow: Database copy '{0}\\{1}' is active. Fail over.", base.Context.Database.Name, base.Context.TargetServerName); new DatabaseFailureItem(FailureNameSpace.ContentIndex, FailureTag.CatalogReseed, base.Context.Database.Guid) { InstanceName = base.Context.Database.Name }.Publish(); return(new AutoReseedCatalogActiveException(base.Context.Database.Name, base.Context.TargetServerName.NetbiosName)); } return(CatalogAutoReseedWorkflow.hookableReseedAction.Value(this)); }
protected bool GetWorkflowElapsedExecutionTime(AutoReseedWorkflowState state, out TimeSpan elapsedExecutionTime) { elapsedExecutionTime = TimeSpan.Zero; if (state.WorkflowExecutionTime.Equals(DateTime.MinValue)) { this.TraceDebug("GetWorkflowElapsedExecutionTime(): Returning 'false' since WorkflowExecutionTime is DateTime.MinValue.", new object[0]); return(false); } elapsedExecutionTime = DateTime.UtcNow.Subtract(state.WorkflowExecutionTime); this.TraceDebug("GetWorkflowElapsedExecutionTime(): Returning elapsedExecutionTime = '{0}'", new object[] { elapsedExecutionTime }); return(true); }
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state) { LocalizedString result = base.RunPrereqs(state); if (!result.IsEmpty) { return(result); } if (base.Context.CopyStatusesForTargetDatabase.Any((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.ActivationPreference < base.Context.TargetCopyStatus.CopyStatus.ActivationPreference)) { AutoReseedWorkflow.Tracer.TraceDebug <string, Guid, string>((long)this.GetHashCode(), "AutoReseed workflow launcher detected failed catalog for database '{0}' [{1}]: {2}. However, the catalog doesn't qualify to be rebuilt.", base.Context.Database.Name, base.Context.Database.Guid, base.Context.TargetCopyStatus.CopyStatus.ErrorMessage); return(ReplayStrings.AutoReseedCatalogSkipRebuild(base.Context.Database.Name, base.Context.TargetServerName.Fqdn)); } return(LocalizedString.Empty); }
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state) { LocalizedString result = base.RunPrereqs(state); if (!result.IsEmpty) { return(result); } result = FailedSuspendedCopyAutoReseedWorkflow.CheckExchangeVolumesPresent(base.Context); if (!result.IsEmpty) { return(result); } return(FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context)); }
public Exception Execute() { Exception ex = null; if (this.IsDisabled) { AutoReseedWorkflow.Tracer.TraceDebug <string, string, Guid>((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}] will not be executed because it has been disabled via regkey.", this.WorkflowName, this.Context.Database.Name, this.Context.Database.Guid); return(null); } try { AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(this.Context.Database.Guid, this.WorkflowType); if (this.ShouldSkipWorkflowExecution(autoReseedWorkflowState)) { return(null); } this.LogWorkflowStarted(); if (this.ArePrereqsSatisfied(autoReseedWorkflowState, out ex) && !this.ShouldThrottleExecution(autoReseedWorkflowState, out ex)) { bool flag = true; try { ex = this.ExecuteInternal(autoReseedWorkflowState); flag = false; } catch (RegistryParameterException ex2) { flag = false; ex = new AutoReseedException(ex2.Message, ex2); } finally { if (flag) { ex = new AutoReseedUnhandledException(this.Context.Database.Name, this.Context.TargetServerName.NetbiosName); } autoReseedWorkflowState.WriteWorkflowExecutionState(ex); } } } catch (RegistryParameterException ex3) { ex = new AutoReseedException(ex3.Message, ex3); } this.LogWorkflowEnded(ex); return(ex); }
// Token: 0x0600194D RID: 6477 RVA: 0x00068814 File Offset: 0x00066A14 protected override TimeSpan GetThrottlingInterval(AutoReseedWorkflowState state) { switch (state.LastReseedRecoveryAction) { case ReseedState.Unknown: case ReseedState.Resume: case ReseedState.Completed: return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedThrottlingIntervalInSecs_Resume)); case ReseedState.AssignSpare: case ReseedState.InPlaceReseed: return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedThrottlingIntervalInSecs_Reseed)); default: return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedThrottlingIntervalInSecs_Resume)); } }
// Token: 0x06001950 RID: 6480 RVA: 0x000689B0 File Offset: 0x00066BB0 protected override Exception ExecuteInternal(AutoReseedWorkflowState state) { Exception result = null; if (state.LastReseedRecoveryAction == ReseedState.Unknown || state.LastReseedRecoveryAction == ReseedState.Resume) { result = this.ExecuteResume(state); } else if (state.LastReseedRecoveryAction == ReseedState.AssignSpare) { result = this.ExecuteAssignSpare(state); } else if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed) { result = this.ExecuteInPlaceReseed(state); } return(result); }
// Token: 0x0600191D RID: 6429 RVA: 0x000675E8 File Offset: 0x000657E8 public static Exception TriggerInPlaceReseed(Guid dbGuid, string dbName) { Exception result = null; try { AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(dbGuid, AutoReseedWorkflowType.FailedSuspendedCopyAutoReseed); autoReseedWorkflowState.ResetReseedRecoveryAction(); autoReseedWorkflowState.IgnoreInPlaceOverwriteDelay = true; autoReseedWorkflowState.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed); ReplayCrimsonEvents.AutoReseedTriggerInPlaceReseed.LogPeriodic <string, Guid>(dbGuid, DiagCore.DefaultEventSuppressionInterval, dbName, dbGuid); } catch (RegistryParameterException ex) { result = ex; } return(result); }
private bool ShouldSkipWorkflowExecution(AutoReseedWorkflowState state) { bool result = false; DateTime workflowNextExecutionTime = state.WorkflowNextExecutionTime; if (workflowNextExecutionTime != DateTime.MinValue && DateTime.UtcNow < workflowNextExecutionTime) { AutoReseedWorkflow.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseed workflow '{0}' for database '{1}' [{2}]: Skipping execution because WorkflowNextExecutionTime is set to '{3}'.", new object[] { this.WorkflowName, this.Context.Database.Name, this.Context.Database.Guid, workflowNextExecutionTime }); result = true; } return(result); }
// Token: 0x06001953 RID: 6483 RVA: 0x00068B44 File Offset: 0x00066D44 private void ResetWorkflowRecoveryActionIfNecessary(AutoReseedWorkflowState state) { TimeSpan timeSpan; if (!base.GetWorkflowElapsedExecutionTime(state, out timeSpan)) { base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Doing nothing.", new object[0]); return; } TimeSpan t = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedWorkflowResetIntervalInSecs); if (!(timeSpan >= t)) { base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Doing nothing since elapsedExecutionTime of '{0}' is too recent.", new object[] { timeSpan }); return; } if (!state.IsLastReseedRecoveryActionPending()) { base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Successfully completed workflow is being reset after duration of '{0}'.", new object[] { timeSpan }); state.ResetReseedRecoveryAction(); return; } if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed && state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedReseedRetryCountMax) { base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Workflow is being reset after duration of '{0}'.", new object[] { timeSpan }); ReplayCrimsonEvents.AutoReseedWorkflowReset.Log <string, Guid, string, string, DateTime>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, state.WorkflowExecutionTime); state.ResetReseedRecoveryAction(); return; } base.TraceDebug("ResetWorkflowRecoveryActionIfNecessary(): Doing nothing since the workflow is still attempting to fix the copy.", new object[0]); }
protected override Exception ExecuteInternal(AutoReseedWorkflowState state) { Exception ex = null; bool flag = false; IVolumeManager volumeManager = base.Context.VolumeManager; IADDatabase database = base.Context.Database; MountedFolderPath databaseMountedFolderPath = VolumeManager.GetDatabaseMountedFolderPath(base.Context.Dag.AutoDagDatabasesRootFolderPath, database.Name); if (MountPointUtil.IsDirectoryAccessibleMountPoint(databaseMountedFolderPath.Path, out ex)) { flag = true; } else { ReplayCrimsonEvents.AutoReseedNeverMountedActiveMissingVolume.Log <string>(database.Name); if (volumeManager.FixActiveDatabaseMountPoint(database, base.Context.Databases, base.Context.AdConfig, out ex, true)) { flag = true; volumeManager.UpdateVolumeInfoCopyState(database.Guid, base.Context.ReplicaInstanceManager); ReplayCrimsonEvents.AutoReseedNeverMountedActiveAllocatedVolume.Log <string, string>(database.Name, base.Context.TargetCopyStatus.CopyStatus.LogVolumeName); } else { AutoReseedWorkflow.Tracer.TraceError <string, string>((long)this.GetHashCode(), "DiskReclaimer: UpdateVolumeForNeverMountedActives() failed to fix up active database: '{0}' mountpoint. Error: {1}", database.Name, AmExceptionHelper.GetExceptionMessageOrNoneString(ex)); ReplayCrimsonEvents.AutoReseedFixActiveMountPointError.Log <string, string>(database.Name, AmExceptionHelper.GetExceptionMessageOrNoneString(ex)); } } if (!flag) { AutoReseedWorkflow.Tracer.TraceError <string, string>((long)this.GetHashCode(), "DiskReclaimer: UpdateVolumeForNeverMountedActives() active database: '{0}' does not have a mountpoint. Skip issuing Store Mount. Error: {1}", database.Name, AmExceptionHelper.GetExceptionMessageOrNoneString(ex)); return(ex); } ex = AmRpcClientHelper.AdminMountDatabaseWrapper(database); if (ex != null) { AutoReseedWorkflow.Tracer.TraceError <string, string>((long)this.GetHashCode(), "DiskReclaimer: UpdateVolumeForNeverMountedActives() failed to mount active database: '{0}' mountpoint. Error: {1}", database.Name, ex.Message); ReplayCrimsonEvents.AutoReseedMountActiveDatabaseError.Log <string, string>(database.Name, ex.Message); } return(ex); }
// Token: 0x06001963 RID: 6499 RVA: 0x0006A044 File Offset: 0x00068244 private Exception ExecuteInPlaceReseed(AutoReseedWorkflowState state) { Exception ex = null; try { state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed); this.LogBeginExecute(state); FailedSuspendedCopyAutoReseedWorkflow.hookableSeedRpc.Value(this); ReplayCrimsonEvents.AutoReseedWorkflowDbFailedBeginReseedSucceeded.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason); } catch (SeederServerException ex2) { ex = ex2; } catch (SeederServerTransientException ex3) { ex = ex3; } this.LogExecuteCompleted(state, ex); return(ex); }
// Token: 0x06001961 RID: 6497 RVA: 0x00069D10 File Offset: 0x00067F10 private Exception ExecuteResume(AutoReseedWorkflowState state) { Exception ex = null; try { state.UpdateReseedRecoveryAction(ReseedState.Resume); this.LogBeginExecute(state); DatabaseCopyActionFlags arg = DatabaseCopyActionFlags.Replication | DatabaseCopyActionFlags.Activation | DatabaseCopyActionFlags.SkipSettingResumeAutoReseedState; FailedSuspendedCopyAutoReseedWorkflow.hookableResumeRpc.Value(AmServerName.LocalComputerName.Fqdn, base.Context.Database.Guid, (uint)arg); } catch (TaskServerException ex2) { ex = ex2; } catch (TaskServerTransientException ex3) { ex = ex3; } this.LogExecuteCompleted(state, ex); return(ex); }
// Token: 0x060018FA RID: 6394 RVA: 0x0006700C File Offset: 0x0006520C private void RunHealthyCopyWorkflowIfNecessary(AutoReseedContext context) { Guid guid = context.Database.Guid; string name = context.Database.Name; bool flag = false; if (context.TargetCopyStatus.CopyStatus.CopyStatus == CopyStatusEnum.Healthy) { try { AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(guid, AutoReseedWorkflowType.FailedSuspendedCopyAutoReseed); AutoReseedWorkflowState autoReseedWorkflowState2 = new AutoReseedWorkflowState(guid, AutoReseedWorkflowType.ManualReseed); AutoReseedWorkflowState autoReseedWorkflowState3 = new AutoReseedWorkflowState(guid, AutoReseedWorkflowType.ManualResume); if (autoReseedWorkflowState2.IsLastReseedRecoveryActionPending() || autoReseedWorkflowState3.IsLastReseedRecoveryActionPending() || autoReseedWorkflowState.IsLastReseedRecoveryActionPending()) { flag = true; if (this.m_suppression.ReportHealthyWorkflowLaunchConditionMet(guid)) { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is Healthy with some prior recovery action having been completed. Launching the HealthyCopyCompletedSeedWorkflow workflow.", name, context.TargetServerName); HealthyCopyCompletedSeedWorkflow healthyCopyCompletedSeedWorkflow = new HealthyCopyCompletedSeedWorkflow(context); healthyCopyCompletedSeedWorkflow.Execute(); } else { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is Healthy with some prior recovery action having been completed but launching the workflow is being skipped due suppression of {2}.", name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbHealthySuppressionInterval); } } } catch (RegistryParameterException arg) { AutoReseedWorkflowLauncher.Tracer.TraceError <string, AmServerName, RegistryParameterException>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Couldn't launch/execute HealthyCopyCompletedSeedWorkflow for database copy '{0}\\{1}' to potentially reset the AutoReseed state. Error: {2}", name, context.TargetServerName, arg); } } if (!flag) { this.m_suppression.ReportHealthyWorkflowNotNeeded(guid); } }
// Token: 0x0600191C RID: 6428 RVA: 0x0006759C File Offset: 0x0006579C public static Exception WriteManualWorkflowExecutionState(Guid dbGuid, AutoReseedWorkflowType manualAction) { Exception result = null; try { AutoReseedWorkflowState autoReseedWorkflowState = new AutoReseedWorkflowState(dbGuid, manualAction); if (manualAction == AutoReseedWorkflowType.ManualResume) { autoReseedWorkflowState.LastReseedRecoveryAction = ReseedState.Resume; } else { autoReseedWorkflowState.LastReseedRecoveryAction = ReseedState.InPlaceReseed; } autoReseedWorkflowState.WriteWorkflowExecutionState(null); } catch (RegistryParameterException ex) { result = ex; } return(result); }
// Token: 0x06001968 RID: 6504 RVA: 0x0006A2E4 File Offset: 0x000684E4 private void LogExecuteCompleted(AutoReseedWorkflowState state, Exception exception) { if (exception == null) { if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.DebugTrace)) { base.TraceDebug("Execution of stage '{0}' succeeded. Attempt number: {1}", new object[] { state.LastReseedRecoveryAction, state.ReseedRecoveryActionRetryCount }); } return; } if (AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.ErrorTrace) || AutoReseedWorkflow.Tracer.IsTraceEnabled(TraceType.ErrorTrace)) { base.TraceError("Execution of stage '{0}' failed with exception: {1}", new object[] { state.LastReseedRecoveryAction, exception }); } ReplayCrimsonEvents.AutoReseedWorkflowDbFailedExecutionStageFailed.Log <string, Guid, string, string, ReseedState, int, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, state.LastReseedRecoveryAction, state.ReseedRecoveryActionRetryCount, exception.Message); }
// Token: 0x06001960 RID: 6496 RVA: 0x00069970 File Offset: 0x00067B70 private LocalizedString RunPrereqsForInPlaceReseed(AutoReseedWorkflowState state) { this.TraceBeginPrereqs(state); int autoReseedDbFailedReseedRetryCountMax = RegistryParameters.AutoReseedDbFailedReseedRetryCountMax; TimeSpan timeSpan = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedSuspendedWorkflowResetIntervalInSecs); timeSpan = this.GetWorkflowSuppressionInterval(timeSpan); if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedReseedRetryCountMax) { state.WriteWorkflowNextExecutionDueTime(timeSpan); base.TraceError("RunPrereqsForInPlaceReseed(): Failing 'InPlaceReseed' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedReseedRetryCountMax ({1}). Workflow will next run after {2}.", new object[] { state.ReseedRecoveryActionRetryCount, autoReseedDbFailedReseedRetryCountMax, timeSpan }); ReplayCrimsonEvents.AutoReseedWorkflowDbReseedRetryExceeded.Log <string, Guid, string, string, int, TimeSpan>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedReseedRetryCountMax, timeSpan); return(ReplayStrings.AutoReseedFailedSeedRetryExceeded(autoReseedDbFailedReseedRetryCountMax)); } if (base.Context.TargetCopyStatus.CopyStatus.InPlaceReseedBlocked) { IEnumerable <CopyStatusClientCachedEntry> enumerable = this.FindTargetDbSetFromDatabaseGroup(base.Context); if (enumerable == null) { enumerable = this.FindTargetDbSetFromNeighbors(base.Context); if (enumerable == null) { return(ReplayStrings.AutoReseedFailedToFindVolumeName); } } MountedFolderPath mountedFolderPath; if (!this.IsVolumeRecentlyAssigned(enumerable, out mountedFolderPath)) { string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage); state.WriteWorkflowNextExecutionDueTime(timeSpan); state.ReseedRecoveryActionRetryCount = RegistryParameters.AutoReseedDbFailedReseedRetryCountMax; base.TraceError("RunPrereqsForInPlaceReseed(): Failing 'InPlaceReseed' prereqs since in-place reseed is blocked. DatabaseCopy has ErrorMessage: {0}. Workflow will next run after {1}.", new object[] { messageOrNoneString, timeSpan }); ReplayCrimsonEvents.AutoReseedWorkflowDbInPlaceReseedBlocked.Log <string, Guid, string, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, messageOrNoneString); return(ReplayStrings.AutoReseedFailedInPlaceReseedBlocked(messageOrNoneString)); } } string edbFilePath = base.Context.Database.EdbFilePath.PathName; if (File.Exists(edbFilePath) && !state.IgnoreInPlaceOverwriteDelay) { DateTime lastWriteTimeUtc = DateTime.MaxValue; Exception ex = MountPointUtil.HandleIOExceptions(delegate { FileInfo fileInfo = new FileInfo(edbFilePath); lastWriteTimeUtc = fileInfo.LastWriteTimeUtc; }); if (ex != null) { lastWriteTimeUtc = base.Context.TargetCopyStatus.CopyStatus.LastStatusTransitionTime; base.TraceError("RunPrereqsForInPlaceReseed(): Failed to read LastWriteTimeUtc of edb file '{0}'. Using LastStatusTransitionTime of '{1}' instead. Exception: {2}", new object[] { edbFilePath, lastWriteTimeUtc, ex }); } TimeSpan timeSpan2 = DateTime.UtcNow.Subtract(lastWriteTimeUtc); TimeSpan timeSpan3 = TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedDbFailedInPlaceReseedDelayInSecs); base.TraceDebug("RunPrereqsForInPlaceReseed(): Found EDB file at '{0}'. The copy was last FailedAndSuspended '{1}' duration ago. Configured reseed delay is: '{2}'.", new object[] { edbFilePath, timeSpan2, timeSpan3 }); if (timeSpan2 <= timeSpan3) { TimeSpan workflowSuppressionInterval = this.GetWorkflowSuppressionInterval(timeSpan3); state.WriteWorkflowNextExecutionDueTime(workflowSuppressionInterval); base.TraceError("RunPrereqsForInPlaceReseed(): EDB file exists and Reseed is being attempted too soon. Reseed will be blocked until the configured reseed delay period. Workflow will next run after {0}.", new object[] { workflowSuppressionInterval }); ReplayCrimsonEvents.AutoReseedWorkflowInPlaceReseedTooSoon.Log <string, Guid, string, string, string, TimeSpan, TimeSpan, TimeSpan>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, edbFilePath, timeSpan2, timeSpan3, workflowSuppressionInterval); return(ReplayStrings.AutoReseedInPlaceReseedTooSoon(timeSpan2.ToString(), timeSpan3.ToString())); } } int num; if (!base.Context.ReseedLimiter.TryStartSeed(out num)) { base.TraceError("RunPrereqsForInPlaceReseed(): Seed is being skipped for now because maximum number of concurrent seeds has been reached: {0}", new object[] { num }); return(ReplayStrings.AutoReseedTooManyConcurrentSeeds(num)); } return(LocalizedString.Empty); }
protected override TimeSpan GetThrottlingInterval(AutoReseedWorkflowState state) { return(TimeSpan.FromSeconds((double)RegistryParameters.AutoReseedCiThrottlingIntervalInSecs)); }
// Token: 0x0600195E RID: 6494 RVA: 0x000696C0 File Offset: 0x000678C0 private LocalizedString RunPrereqsForAssignSpare(AutoReseedWorkflowState state) { this.TraceBeginPrereqs(state); if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax) { int autoReseedDbFailedAssignSpareRetryCountMax = RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax; base.TraceError("RunPrereqsForAssignSpare(): Failing 'AssignSpare' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedAssignSpareRetryCountMax ({1}). Workflow will try InPlaceReseed stage next.", new object[] { state.ReseedRecoveryActionRetryCount, autoReseedDbFailedAssignSpareRetryCountMax }); ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRetryExceeded.Log <string, Guid, string, string, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedAssignSpareRetryCountMax); state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed); return(ReplayStrings.AutoReseedFailedSeedRetryExceeded(autoReseedDbFailedAssignSpareRetryCountMax)); } state.UpdateReseedRecoveryAction(ReseedState.AssignSpare); if (string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName) || string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName)) { return(ReplayStrings.AutoReseedFailedToFindTargetVolumeName(AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.VolumeInfoLastError))); } MountedFolderPath mountedFolderPath = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName); MountedFolderPath other = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName); if (!mountedFolderPath.Equals(other)) { return(ReplayStrings.AutoReseedLogAndDbNotOnSameVolume); } IEnumerable <CopyStatusClientCachedEntry> enumerable = this.FindTargetDbSetFromDatabaseGroup(base.Context); if (enumerable == null) { enumerable = this.FindTargetDbSetFromNeighbors(base.Context); if (enumerable == null) { return(ReplayStrings.AutoReseedFailedToFindVolumeName); } } MountedFolderPath mountedFolderPath2; if (this.IsVolumeRecentlyAssigned(enumerable, out mountedFolderPath2)) { ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRecentlyAssigned.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason); state.AssignedVolumeName = mountedFolderPath2.Path; state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed); } else if (this.DoesMountPointNeedToBeFixed(enumerable, out mountedFolderPath2)) { base.TraceDebug("Database copy is missing a mount point, and it will be fixed up to point to volume '{0}'", new object[] { mountedFolderPath2 }); this.m_volumeForMissingMountPoint = mountedFolderPath2; } else { string databaseNames = FailedSuspendedCopyAutoReseedWorkflow.GetDatabaseNames(enumerable); if (!enumerable.All((CopyStatusClientCachedEntry status) => !status.IsActive)) { return(ReplayStrings.AutoReseedNotAllCopiesPassive(databaseNames)); } if (!enumerable.All((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.CopyStatus == CopyStatusEnum.FailedAndSuspended)) { return(ReplayStrings.AutoReseedNotAllCopiesOnVolumeFailedSuspended(databaseNames)); } } this.m_targetDbSet = enumerable; return(LocalizedString.Empty); }
protected override TimeSpan GetThrottlingInterval(AutoReseedWorkflowState state) { return(TimeSpan.Zero); }