// Token: 0x0600194F RID: 6479 RVA: 0x0006886C File Offset: 0x00066A6C protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state) { LocalizedString result = base.RunPrereqs(state); if (!result.IsEmpty) { return(result); } result = FailedSuspendedCopyAutoReseedWorkflow.hookableCheckExVolumes.Value(base.Context); if (!result.IsEmpty) { return(result); } result = FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context); if (!result.IsEmpty) { return(result); } if (base.Context.TargetCopyStatus.CopyStatus.ActionInitiator == ActionInitiatorType.Administrator) { base.TraceError("RunPrereqs(): The DatabaseCopy has been suspended by an Administrator so AutoReseed will not be attempted.", new object[0]); return(ReplayStrings.AutoReseedFailedAdminSuspended); } if (base.Context.TargetCopyStatus.CopyStatus.ReseedBlocked) { string messageOrNoneString = AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.ErrorMessage); base.TraceError("RunPrereqs(): The DatabaseCopy is marked as ReseedBlocked so AutoReseed will not be attempted. Database copy ErrorMessage: {0}", new object[] { messageOrNoneString }); return(ReplayStrings.AutoReseedFailedReseedBlocked(messageOrNoneString)); } this.ResetWorkflowRecoveryActionIfNecessary(state); if (!state.IsLastReseedRecoveryActionPending()) { base.TraceDebug("RunPrereqs(): Running the workflow for the first time, so starting with the Resume action. LastReseedRecoveryAction = {0}", new object[] { state.LastReseedRecoveryAction }); state.UpdateReseedRecoveryAction(ReseedState.Resume); } if (state.LastReseedRecoveryAction == ReseedState.Resume) { result = this.RunPrereqsForResume(state); } if (state.LastReseedRecoveryAction == ReseedState.AssignSpare) { result = this.RunPrereqsForAssignSpare(state); } if (state.LastReseedRecoveryAction == ReseedState.InPlaceReseed) { result = this.RunPrereqsForInPlaceReseed(state); } return(result); }
protected override LocalizedString RunPrereqs(AutoReseedWorkflowState state) { LocalizedString result = base.RunPrereqs(state); if (!result.IsEmpty) { return(result); } result = FailedSuspendedCopyAutoReseedWorkflow.CheckExchangeVolumesPresent(base.Context); if (!result.IsEmpty) { return(result); } return(FailedSuspendedCopyAutoReseedWorkflow.CheckDatabaseLogPaths(base.Context)); }
// Token: 0x06001955 RID: 6485 RVA: 0x00068D58 File Offset: 0x00066F58 private bool DoesMountPointNeedToBeFixed(IEnumerable <CopyStatusClientCachedEntry> targetDbSet, out MountedFolderPath volumeName) { volumeName = MountedFolderPath.Empty; RpcDatabaseCopyStatus2 copyStatus = base.Context.TargetCopyStatus.CopyStatus; if (FailedSuspendedCopyAutoReseedWorkflow.DoesCopyHaveMountPointConfigured(copyStatus)) { return(false); } CopyStatusClientCachedEntry copyStatusClientCachedEntry = targetDbSet.FirstOrDefault((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && FailedSuspendedCopyAutoReseedWorkflow.DoesCopyHaveMountPointConfigured(status.CopyStatus)); if (copyStatusClientCachedEntry == null) { base.TraceError("Database copy is missing a mount point, but no other copies in the grouping have mount points either.", new object[0]); return(false); } volumeName = new MountedFolderPath(copyStatusClientCachedEntry.CopyStatus.DatabaseVolumeName); return(true); }
// Token: 0x06001964 RID: 6500 RVA: 0x0006A0D8 File Offset: 0x000682D8 private static void BeginSeedRpc(FailedSuspendedCopyAutoReseedWorkflow wf) { using (SeederClient seederClient = SeederClient.Create(wf.Context.TargetServerName.Fqdn, wf.Context.Database.Name, null, wf.Context.TargetServer.AdminDisplayVersion)) { try { seederClient.CancelDbSeed(wf.Context.Database.Guid); seederClient.EndDbSeed(wf.Context.Database.Guid); wf.TraceDebug("Prior seed instance has been Cancelled.", new object[0]); } catch (SeederInstanceNotFoundException ex) { wf.TraceDebug("Prior seed instance cleanup failed with: {0}", new object[] { ex }); } seederClient.PrepareDbSeedAndBegin(wf.Context.Database.Guid, false, true, false, false, true, true, string.Empty, null, string.Empty, null, null, SeederRpcFlags.SkipSettingReseedAutoReseedState); } }
// Token: 0x06001958 RID: 6488 RVA: 0x00069248 File Offset: 0x00067448 private IEnumerable <CopyStatusClientCachedEntry> FindTargetDbSetFromNeighbors(AutoReseedContext context) { IEnumerable <CopyStatusClientCachedEntry> enumerable = null; IEnumerable <CopyStatusClientCachedEntry> enumerable2 = null; Dictionary <AmServerName, IEnumerable <CopyStatusClientCachedEntry> > copyStatusesForDag = context.CopyStatusesForDag; IEnumerable <KeyValuePair <AmServerName, IEnumerable <CopyStatusClientCachedEntry> > > enumerable3 = from kvp in copyStatusesForDag let server = kvp.Key let isTargetServer = this.IsTargetServer(server, context.TargetServerName) where this.ShouldSelectServer(isTargetServer) orderby isTargetServer descending select kvp; foreach (KeyValuePair <AmServerName, IEnumerable <CopyStatusClientCachedEntry> > keyValuePair in enumerable3) { AmServerName key = keyValuePair.Key; IEnumerable <CopyStatusClientCachedEntry> value = keyValuePair.Value; IEnumerable <CopyStatusClientCachedEntry> source = from status in value where status.Result == CopyStatusRpcResult.Success && FailedSuspendedCopyAutoReseedWorkflow.DoesCopyHaveMountPointConfigured(status.CopyStatus) select status; IEnumerable <IGrouping <string, CopyStatusClientCachedEntry> > databaseSets = source.GroupBy((CopyStatusClientCachedEntry status) => status.CopyStatus.DatabaseVolumeName, StringComparer.OrdinalIgnoreCase); IGrouping <string, CopyStatusClientCachedEntry> grouping = FailedSuspendedCopyAutoReseedWorkflow.FindDatabaseGrouping(context, databaseSets); if (grouping != null) { int num = grouping.Count <CopyStatusClientCachedEntry>(); if (num == context.Dag.AutoDagDatabaseCopiesPerVolume) { enumerable2 = grouping; break; } } } if (enumerable2 != null) { Dictionary <Guid, CopyStatusClientCachedEntry> remoteDbMap = enumerable2.ToDictionary((CopyStatusClientCachedEntry status) => status.DbGuid); enumerable = from status in context.CopyStatusesForTargetServer where remoteDbMap.ContainsKey(status.DbGuid) select status; enumerable = enumerable.ToArray <CopyStatusClientCachedEntry>(); } return(enumerable); }
// Token: 0x060018F8 RID: 6392 RVA: 0x00066668 File Offset: 0x00064868 public void BeginAutoReseedIfNecessary(AutoReseedContext context) { Guid guid = context.Database.Guid; string name = context.Database.Name; if (context.TargetCopyStatus == null) { AutoReseedWorkflowLauncher.Tracer.TraceError <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' has null TargetCopyStatus. Skipping.", name, context.TargetServerName); return; } if (context.TargetCopyStatus.Result != CopyStatusRpcResult.Success) { AutoReseedWorkflowLauncher.Tracer.TraceError((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Skipping since GetCopyStatus RPC to database copy '{0}\\{1}' failed. Result: {2}. Error: {3}.", new object[] { name, context.TargetServerName, context.TargetCopyStatus.Result, context.TargetCopyStatus.LastException }); return; } this.RunNeverMountedActiveWorkflow(context); this.RunHealthyCopyWorkflowIfNecessary(context); ExtendedErrorInfo extendedErrorInfo = context.TargetCopyStatus.CopyStatus.ExtendedErrorInfo; if (context.TargetCopyStatus.CopyStatus.CopyStatus == CopyStatusEnum.Failed && (extendedErrorInfo == null || extendedErrorInfo.FailureException == null || !(extendedErrorInfo.FailureException is ReplayServiceRpcCopyStatusTimeoutException))) { if (context.TargetCopyStatus.IsActive) { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is 'Failed' but active. Skipping.", name, context.TargetServerName); return; } if (this.m_suppression.ReportWorkflowLaunchConditionMet(AutoReseedWorkflowType.FailedCopy, guid, CatalogAutoReseedWorkflow.CatalogAutoReseedReason.None, 1)) { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' has been 'Failed' for at least {2}. Launching the FailedCopyWorkflow workflow.", name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbFailedSuppresionInterval); FailedCopyWorkflow failedCopyWorkflow = new FailedCopyWorkflow(context, context.TargetCopyStatus.CopyStatus.ErrorMessage); failedCopyWorkflow.Execute(); return; } AutoReseedWorkflowLauncher.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is 'Failed' but launching the recovery workflow is being skipped due to either initial suppression of {2}, or periodic suppression of {3}.", new object[] { name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbFailedSuppresionInterval, AutoReseedWorkflowSuppression.s_dbFailedSuppresionInterval }); return; } else { if (context.TargetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.FailedAndSuspended) { if (context.TargetCopyStatus.CopyStatus.ContentIndexStatus == ContentIndexStatusType.FailedAndSuspended) { if (context.CopyStatusesForTargetDatabase.All((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && (status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Disabled || status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.Suspended || status.CopyStatus.ContentIndexStatus == ContentIndexStatusType.FailedAndSuspended))) { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, Guid, string>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: AutoReseed workflow launcher detected all catalogs failed for database '{0}' [{1}]: {2}.", context.Database.Name, context.Database.Guid, context.TargetCopyStatus.CopyStatus.ErrorMessage); ReplayCrimsonEvents.AutoReseedWorkflowAllCatalogFailed.Log <string, Guid, string, string>(context.Database.Name, context.Database.Guid, "FailedSuspendedCatalogRebuildWorkflow", context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage); if (this.m_suppression.ReportWorkflowLaunchConditionMet(AutoReseedWorkflowType.FailedSuspendedCatalogRebuild, guid, CatalogAutoReseedWorkflow.CatalogAutoReseedReason.None, 1)) { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database '{0}' has no catalogs in healthy state for at least {1}. Launching the recovery workflow.", name, AutoReseedWorkflowSuppression.s_ciRebuildSuppresionInterval); FailedSuspendedCatalogRebuildWorkflow failedSuspendedCatalogRebuildWorkflow = new FailedSuspendedCatalogRebuildWorkflow(context, context.TargetCopyStatus.CopyStatus.ContentIndexErrorMessage); failedSuspendedCatalogRebuildWorkflow.Execute(); return; } AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, TimeSpan, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database '{0}' has no catalogs in healthy state but launching the recovery workflow is being skipped due to either initial suppression of {1}, or periodic suppression of {2}.", name, AutoReseedWorkflowSuppression.s_ciRebuildSuppresionInterval, AutoReseedWorkflowSuppression.s_ciRebuildRetryInterval); return; } } if (!this.TryLaunchCatalogAutoReseedWorkflow(context, name)) { this.m_suppression.ReportNoWorkflowsNeedToLaunch(guid); } return; } if (context.TargetCopyStatus.IsActive) { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is 'FailedAndSuspended' but active. Skipping.", name, context.TargetServerName); return; } if (this.m_suppression.ReportWorkflowLaunchConditionMet(AutoReseedWorkflowType.FailedSuspendedCopyAutoReseed, guid, CatalogAutoReseedWorkflow.CatalogAutoReseedReason.None, 1)) { AutoReseedWorkflowLauncher.Tracer.TraceDebug <string, AmServerName, TimeSpan>((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' has been FailedAndSuspended for at least {2}. Launching the recovery workflow.", name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbReseedSuppresionInterval); FailedSuspendedCopyAutoReseedWorkflow failedSuspendedCopyAutoReseedWorkflow = new FailedSuspendedCopyAutoReseedWorkflow(context, context.TargetCopyStatus.CopyStatus.ErrorMessage); failedSuspendedCopyAutoReseedWorkflow.Execute(); return; } AutoReseedWorkflowLauncher.Tracer.TraceDebug((long)this.GetHashCode(), "AutoReseedWorkflowLauncher.BeginAutoReseedIfNecessary: Database copy '{0}\\{1}' is FailedAndSuspended but launching the recovery workflow is being skipped due to either initial suppression of {2}, or periodic suppression of {3}.", new object[] { name, context.TargetServerName, AutoReseedWorkflowSuppression.s_dbReseedSuppresionInterval, AutoReseedWorkflowSuppression.s_dbReseedRetryInterval }); return; } }
// Token: 0x0600195E RID: 6494 RVA: 0x000696C0 File Offset: 0x000678C0 private LocalizedString RunPrereqsForAssignSpare(AutoReseedWorkflowState state) { this.TraceBeginPrereqs(state); if (state.ReseedRecoveryActionRetryCount >= RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax) { int autoReseedDbFailedAssignSpareRetryCountMax = RegistryParameters.AutoReseedDbFailedAssignSpareRetryCountMax; base.TraceError("RunPrereqsForAssignSpare(): Failing 'AssignSpare' prereqs since ReseedRecoveryActionRetryCount ({0}) exceeds AutoReseedDbFailedAssignSpareRetryCountMax ({1}). Workflow will try InPlaceReseed stage next.", new object[] { state.ReseedRecoveryActionRetryCount, autoReseedDbFailedAssignSpareRetryCountMax }); ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRetryExceeded.Log <string, Guid, string, string, int>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason, autoReseedDbFailedAssignSpareRetryCountMax); state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed); return(ReplayStrings.AutoReseedFailedSeedRetryExceeded(autoReseedDbFailedAssignSpareRetryCountMax)); } state.UpdateReseedRecoveryAction(ReseedState.AssignSpare); if (string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName) || string.IsNullOrEmpty(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName)) { return(ReplayStrings.AutoReseedFailedToFindTargetVolumeName(AmExceptionHelper.GetMessageOrNoneString(base.Context.TargetCopyStatus.CopyStatus.VolumeInfoLastError))); } MountedFolderPath mountedFolderPath = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.LogVolumeName); MountedFolderPath other = new MountedFolderPath(base.Context.TargetCopyStatus.CopyStatus.DatabaseVolumeName); if (!mountedFolderPath.Equals(other)) { return(ReplayStrings.AutoReseedLogAndDbNotOnSameVolume); } IEnumerable <CopyStatusClientCachedEntry> enumerable = this.FindTargetDbSetFromDatabaseGroup(base.Context); if (enumerable == null) { enumerable = this.FindTargetDbSetFromNeighbors(base.Context); if (enumerable == null) { return(ReplayStrings.AutoReseedFailedToFindVolumeName); } } MountedFolderPath mountedFolderPath2; if (this.IsVolumeRecentlyAssigned(enumerable, out mountedFolderPath2)) { ReplayCrimsonEvents.AutoReseedWorkflowDbSpareRecentlyAssigned.Log <string, Guid, string, string>(base.Context.Database.Name, base.Context.Database.Guid, base.WorkflowName, base.WorkflowLaunchReason); state.AssignedVolumeName = mountedFolderPath2.Path; state.UpdateReseedRecoveryAction(ReseedState.InPlaceReseed); } else if (this.DoesMountPointNeedToBeFixed(enumerable, out mountedFolderPath2)) { base.TraceDebug("Database copy is missing a mount point, and it will be fixed up to point to volume '{0}'", new object[] { mountedFolderPath2 }); this.m_volumeForMissingMountPoint = mountedFolderPath2; } else { string databaseNames = FailedSuspendedCopyAutoReseedWorkflow.GetDatabaseNames(enumerable); if (!enumerable.All((CopyStatusClientCachedEntry status) => !status.IsActive)) { return(ReplayStrings.AutoReseedNotAllCopiesPassive(databaseNames)); } if (!enumerable.All((CopyStatusClientCachedEntry status) => status.Result == CopyStatusRpcResult.Success && status.CopyStatus.CopyStatus == CopyStatusEnum.FailedAndSuspended)) { return(ReplayStrings.AutoReseedNotAllCopiesOnVolumeFailedSuspended(databaseNames)); } } this.m_targetDbSet = enumerable; return(LocalizedString.Empty); }
internal bool DetectLowSpace(IADDatabase db, IMonitoringADConfig adConfig) { if (db.ReplicationType != ReplicationType.Remote) { return(false); } if (!FailedSuspendedCopyAutoReseedWorkflow.IsAutoReseedEnabled(db)) { return(false); } int num = RegistryParameters.SpaceMonitorMinHealthyCount; DatabaseRedundancyValidator databaseRedundancyValidator = new DatabaseRedundancyValidator(db, num, this.m_statusLookup, adConfig, null, true); IHealthValidationResult healthValidationResult = databaseRedundancyValidator.Run(); CopyStatusClientCachedEntry targetCopyStatus = healthValidationResult.TargetCopyStatus; if (targetCopyStatus == null || targetCopyStatus.CopyStatus == null || targetCopyStatus.Result != CopyStatusRpcResult.Success || targetCopyStatus.CopyStatus.CopyStatus != CopyStatusEnum.Healthy || targetCopyStatus.CopyStatus.DiskTotalSpaceBytes == 0UL) { return(false); } if (targetCopyStatus.CopyStatus.LastLogInfoIsStale || (targetCopyStatus.CopyStatus.GetCopyQueueLength() < (long)RegistryParameters.SpaceMonitorCopyQueueThreshold && targetCopyStatus.CopyStatus.GetReplayQueueLength() < (long)RegistryParameters.SpaceMonitorReplayQueueThreshold)) { return(false); } if (healthValidationResult.IsTargetCopyHealthy) { num++; } long num2 = (long)(targetCopyStatus.CopyStatus.DiskFreeSpaceBytes / 1048576UL); long num3 = (long)(targetCopyStatus.CopyStatus.DiskTotalSpaceBytes / 1048576UL); long num4 = 0L; bool flag = false; if (num2 <= targetCopyStatus.CopyStatus.GetCopyQueueLength() || num3 <= 0L) { flag = true; } else { num4 = (num2 - targetCopyStatus.CopyStatus.GetCopyQueueLength()) * 1048576L / 1048576L; if (num4 <= (long)RegistryParameters.SpaceMonitorLowSpaceThresholdInMB) { flag = true; } } SpaceMonitor.Tracer.TraceDebug((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Database '{0}' has {1}MB remaining = {2}MB effective. MinHealthy={3} ReportedHealthy={4}", new object[] { db.Name, num2, num4, num, healthValidationResult.HealthyCopiesCount }); if (!flag) { this.m_actionSuppression.ReportSuccess(db.Guid); return(false); } bool flag2 = !this.m_actionSuppression.ReportFailure(db.Guid, this.SuppressionWindow); SpaceMonitor.Tracer.TraceError <string, bool>((long)this.GetHashCode(), "SpaceMonitor.ProcessDatabase: Local copy of Database '{0}' is low on space. Action suppressed = {1}", db.Name, flag2); if (flag2) { return(false); } ReplayCrimsonEvents.SpaceMonitorLowSpaceDetected.LogPeriodic <string, Guid, long, long, string, long, long, long, long, int, int>(db.Guid, this.m_lowSpaceLoggingPeriod, db.Name, db.Guid, num3, num2, string.Format("{0:0.000}", (double)num2 / (double)num3 * 100.0), targetCopyStatus.CopyStatus.GetCopyQueueLength(), targetCopyStatus.CopyStatus.GetReplayQueueLength(), targetCopyStatus.CopyStatus.LowestLogPresent, targetCopyStatus.CopyStatus.LastLogCopied, healthValidationResult.HealthyCopiesCount, num); return(healthValidationResult.HealthyCopiesCount >= num); }