// Token: 0x060004A4 RID: 1188 RVA: 0x0001907C File Offset: 0x0001727C internal static void DismountAll(string hint) { AmTrace.Debug("AmStoreHelper.DismountAll( {0} ): Started dismounting all mounted databases.", new object[] { hint }); lock (AmStoreHelper.ForceDismountLocker) { MdbStatus[] array = null; DateTime utcNow = DateTime.UtcNow; if (!AmStoreHelper.GetAllDatabaseStatuses(null, false, out array)) { AmTrace.Error("AmStoreHelper.DismountAll( {0} ): GetAllDatabaseStatuses() failed. Now attempting to kill store to quickly get dismounted databases.", new object[] { hint }); AmStoreServiceMonitor.KillStoreIfRunningBefore(utcNow, "DismountAll"); } else if (array != null && array.Length > 0) { DismountDatabasesInParallel dismountDatabasesInParallel = new DismountDatabasesInParallel(array); dismountDatabasesInParallel.Execute(RegistryParameters.AmDismountOrKillTimeoutInSec * 1000, hint); } else { AmTrace.Debug("Dismount all skipped since there are no mounted databases", new object[0]); } } }
private void ReportStart() { AmConfig config = AmSystemManager.Instance.Config; this.ReportStoreStartedToReplayManager(); if (config.IsUnknown) { AmServiceMonitor.Tracer.TraceError(0L, "Store service start detected, but configuration is in unknown state"); return; } if (config.IsStandalone) { AmStoreServiceMonitor.ReportStoreStatus(AmServerName.LocalComputerName, AmSystemEventCode.StoreServiceStarted, AmServerName.LocalComputerName); this.m_isStartReported = true; return; } AmServerName currentPAM = config.DagConfig.CurrentPAM; if (config.DagConfig.IsNodePubliclyUp(currentPAM)) { AmStoreServiceMonitor.ReportStoreStatus(currentPAM, AmSystemEventCode.StoreServiceStarted, AmServerName.LocalComputerName); this.m_isStartReported = true; return; } AmServiceMonitor.Tracer.TraceInformation <AmServerName>(0, 0L, "Store service monitor is not reporting the store start to the PAM, since PAM on '{0}'is not up yet. Store monitor will retry once the node is up", currentPAM); }
// Token: 0x060004B1 RID: 1201 RVA: 0x000194CC File Offset: 0x000176CC public bool Execute(int waitTimeoutMs, string hint) { bool flag = false; Stopwatch stopwatch = new Stopwatch(); AmStoreHelper.DismountDelegate dismountDelegate = new AmStoreHelper.DismountDelegate(AmStoreHelper.RemoteDismount); this.m_completedEvent = new ManualResetEvent(false); stopwatch.Start(); DateTime utcNow = DateTime.UtcNow; try { ReplayCrimsonEvents.ForceDismountingDatabases.Log <AmServerName, string>(AmServerName.LocalComputerName, hint); if (this.m_mdbStatuses != null && this.m_mdbStatuses.Length > 0) { AmTrace.Debug("DismountDatabasesInParallel.Execute() now starting with timeout of {0} ms...", new object[] { waitTimeoutMs }); foreach (MdbStatus mdbStatus in this.m_mdbStatuses) { DismountDatabasesInParallel.AsyncDismountState @object = new DismountDatabasesInParallel.AsyncDismountState(mdbStatus.MdbGuid, dismountDelegate); dismountDelegate.BeginInvoke(null, mdbStatus.MdbGuid, UnmountFlags.SkipCacheFlush, false, new AsyncCallback(this.DismountCompletionCallback), @object); } if (this.m_completedEvent.WaitOne(waitTimeoutMs)) { AmTrace.Debug("DismountDatabasesInParallel.Execute() finished dismounting DBs in {0} ms.", new object[] { stopwatch.ElapsedMilliseconds }); flag = true; } else { AmTrace.Error("DismountDatabasesInParallel.Execute() timed out waiting for DBs to finish dismounting.", new object[0]); AmStoreServiceMonitor.KillStoreIfRunningBefore(utcNow, "DismountDatabasesInParallel"); } } } finally { ReplayCrimsonEvents.ForceDismountAllDatabasesComplete.Log <bool>(flag); lock (this.m_locker) { this.m_completedEvent.Close(); this.m_completedEvent = null; } } return(flag); }
protected override void OnStop() { this.m_isStopReported = false; this.m_isStopObserved = true; AmTrace.Debug("AmStoreServiceMonitor.OnStop clearing AM counters.", new object[0]); ExTraceGlobals.FaultInjectionTracer.TraceTest(3764792637U); foreach (string instanceName in ActiveManagerPerfmon.GetInstanceNames()) { if (!AmStoreServiceMonitor.IsTotalInstanceName(instanceName)) { ActiveManagerPerfmon.ResetInstance(instanceName); } } this.OnWaitingForStart(); }
public static Exception KillStoreIfRunningBefore(DateTime limitTimeUtc, string reason) { Exception ex = null; if (AmStoreServiceMonitor.s_lastKillTimeUtc != null && AmStoreServiceMonitor.s_lastKillTimeUtc >= limitTimeUtc) { AmServiceMonitor.Tracer.TraceDebug <DateTime>(0L, "KillStoreIfRunningBefore ignores Kill since store last killed at {0}UTC", AmStoreServiceMonitor.s_lastKillTimeUtc.Value); return(ex); } if (Interlocked.CompareExchange(ref AmStoreServiceMonitor.s_numThreadsInStoreKill, 1, 0) == 1) { AmServiceMonitor.Tracer.TraceDebug(0L, "KillStoreIfRunningBefore ignores Kill since another thread is currently killing store."); return(ex); } Exception result; try { AmServiceMonitor.Tracer.TraceDebug <string>(0L, "KillStoreIfRunningBefore killing now. Reason: {0}", reason); ex = AmStoreServiceMonitor.TryCrashingStoreGracefully(); if (ex != null) { ReplayEventLogConstants.Tuple_AmFailedToStopService.LogEvent(null, new object[] { "MSExchangeIS", ex.ToString(), reason }); } else { ReplayEventLogConstants.Tuple_AmKilledStoreToForceDismount.LogEvent(null, new object[] { reason }); } AmStoreServiceMonitor.s_lastKillTimeUtc = new DateTime?(DateTime.UtcNow); result = ex; } finally { if (Interlocked.CompareExchange(ref AmStoreServiceMonitor.s_numThreadsInStoreKill, 0, 1) == 0) { DiagCore.RetailAssert(false, "We should not have more than 1 thread in KillStore()", new object[0]); } } return(result); }
private static void ReportKillStarted() { AmStoreServiceMonitor.s_killWasTriggered = true; AmSystemEventCode eventCode = AmSystemEventCode.StoreServiceUnexpectedlyStopped; AmConfig config = AmSystemManager.Instance.Config; if (!config.IsUnknown && !config.IsStandalone) { AmServerName currentPAM = config.DagConfig.CurrentPAM; if (config.DagConfig.IsNodePubliclyUp(currentPAM)) { AmTrace.Diagnostic("Reporting to PAM ({0}) that store process is being killed.", new object[] { currentPAM }); AmStoreServiceMonitor.ReportStoreStatus(currentPAM, eventCode, AmServerName.LocalComputerName); } } }
// Token: 0x060004A9 RID: 1193 RVA: 0x00019340 File Offset: 0x00017540 private static Exception DismountWithKillOnTimeout(Guid mdbGuid, UnmountFlags flags, bool retryOnConflict) { AmTrace.Debug("DismountWithKillOnTimeout {0}", new object[] { mdbGuid }); Exception result = null; bool flag = false; DateTime utcNow = DateTime.UtcNow; TimeSpan timeSpan = TimeSpan.FromSeconds((double)RegistryParameters.AmDismountOrKillTimeoutInSec); try { InvokeWithTimeout.Invoke(delegate() { AmStoreHelper.RemoteDismount(null, mdbGuid, flags, retryOnConflict); }, timeSpan); } catch (TimeoutException) { flag = true; } catch (MapiPermanentException ex) { result = ex; } catch (MapiRetryableException ex2) { result = ex2; } if (flag) { AmTrace.Debug("Dismount {0} timedOut after {1}ms", new object[] { mdbGuid, timeSpan.TotalMilliseconds }); ReplayCrimsonEvents.DismountFailedOnTimeout.Log <Guid, TimeSpan>(mdbGuid, timeSpan); result = AmStoreServiceMonitor.KillStoreIfRunningBefore(utcNow, "DismountWithKillOnTimeout"); } return(result); }
internal static Exception TryCrashingStoreGracefully() { Exception ex = null; Process storeProcess = null; EventWaitHandle crashControlAckEvent = null; try { ReplayCrimsonEvents.InitiatingGracefulStoreCrash.Log(); storeProcess = ServiceOperations.GetServiceProcess("MSExchangeIS", out ex); if (ex == null) { ex = ServiceOperations.RunOperation(delegate(object param0, EventArgs param1) { if (!RegistryParameters.KillStoreInsteadOfWatsonOnTimeout) { crashControlAckEvent = new EventWaitHandle(false, EventResetMode.ManualReset, "Global\\17B584B2-A9E0-45CF-87CB-7774112D6CB9"); ThreadPool.QueueUserWorkItem(delegate(object param0) { Exception ex2 = ServiceOperations.ControlService("MSExchangeIS", 130); if (ex2 != null) { AmTrace.Debug("ControlService() failed with {0}", new object[] { ex2.Message }); } }); } else { AmTrace.Diagnostic("Killing store instead of taking a Watson dump due to registry override.", new object[0]); } if (crashControlAckEvent != null) { if (crashControlAckEvent.WaitOne(RegistryParameters.StoreCrashControlCodeAckTimeoutInMSec)) { AmStoreServiceMonitor.ReportKillStarted(); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); if (!storeProcess.WaitForExit(RegistryParameters.StoreWatsonDumpTimeoutInMSec)) { AmTrace.Diagnostic("Store process did not finish taking dump in {0} msecs", new object[] { RegistryParameters.StoreWatsonDumpTimeoutInMSec }); } else { AmTrace.Diagnostic("Store process finished taking dump in {0} msecs", new object[] { stopwatch.Elapsed.TotalMilliseconds }); } } else { AmTrace.Diagnostic("Store failed to acknowledge that it received the crash control code in {0} msecs.", new object[] { RegistryParameters.StoreCrashControlCodeAckTimeoutInMSec }); AmStoreServiceMonitor.ReportKillStarted(); } } else { AmStoreServiceMonitor.ReportKillStarted(); } if (!storeProcess.HasExited) { if (crashControlAckEvent != null) { AmTrace.Diagnostic("Store process is still running even after the graceful attempt. Force killing it.", new object[0]); } storeProcess.Kill(); TimeSpan timeSpan = TimeSpan.FromMilliseconds((double)RegistryParameters.StoreKillBugcheckTimeoutInMSec); if (!storeProcess.WaitForExit(RegistryParameters.StoreKillBugcheckTimeoutInMSec)) { ExDateTime storeKillBugcheckDisabledTime = RegistryParameters.StoreKillBugcheckDisabledTime; string text = string.Format("Store process is still running {0} secs after attempt to force kill it.", timeSpan.TotalSeconds); if (storeKillBugcheckDisabledTime > ExDateTime.UtcNow) { AmTrace.Debug("Store bugcheck has been disabled by regkey '{0}' until '{1}'.", new object[] { "StoreKillBugcheckDisabledTime", storeKillBugcheckDisabledTime }); ReplayCrimsonEvents.StoreBugCheckDisabledUntilTime.LogPeriodic <string, string, ExDateTime>(Environment.MachineName, DiagCore.DefaultEventSuppressionInterval, text, "StoreKillBugcheckDisabledTime", storeKillBugcheckDisabledTime); return; } AmTrace.Debug("Attempting to bugcheck the system. Reason: {0}", new object[] { text }); BugcheckHelper.TriggerBugcheckIfRequired(DateTime.UtcNow, text); return; } else { AmTrace.Diagnostic("Store process has been forcefully killed.", new object[0]); } } }); } } finally { ReplayCrimsonEvents.FinishedGracefulStoreCrash.Log <string>((ex != null) ? ex.Message : "<none>"); if (crashControlAckEvent != null) { crashControlAckEvent.Close(); } if (storeProcess != null) { storeProcess.Dispose(); } } return(ex); }