private void Open() { if (this.m_logShipContext != null && !this.m_logShipContext.IsInvalid) { return; } uint hr = 0U; uint tmpDwExtError = 0U; CLogShipContext tmpLogShipContext = null; uint rpcTimeoutMsec = (uint)this.m_timeout.TotalMilliseconds; string guid = this.m_config.IdentityGuid.ToString(); Action operation = delegate() { hr = CLogShipContext.Open(this.m_config.SourceMachine, guid, this.m_config.ServerName, this.m_config.LogFilePrefix, this.m_config.DestinationLogPath, this.m_config.CircularLoggingEnabled, ReplicaType.StandbyReplica, rpcTimeoutMsec, out tmpDwExtError, out tmpLogShipContext); if (hr == 0U) { this.m_logShipContext = tmpLogShipContext; } }; try { base.ProtectedCallWithTimeout("LogShipOpenContext", this.m_timeout, operation); if (hr != 0U || tmpLogShipContext == null || tmpLogShipContext.IsInvalid) { if (hr == 3355379665U) { throw new CopyUnknownToActiveLogTruncationException(guid, this.m_config.SourceMachine, this.m_config.ServerName, hr); } string text; if (hr == 3355444321U) { text = ReplayStrings.FileSystemCorruptionDetected(this.m_config.DestinationLogPath); ReplayEventLogConstants.Tuple_FilesystemCorrupt.LogEvent(this.m_config.IdentityGuid.ToString(), new object[] { this.m_config.DestinationLogPath }); FailureItemPublisherHelper.PublishAction(FailureTag.FileSystemCorruption, this.m_config.IdentityGuid, this.m_config.IdentityGuid.ToString()); } else { text = SeedHelper.TranslateEsebackErrorCode((long)((ulong)hr), (long)((ulong)tmpDwExtError)); if (text == null) { text = ReplayStrings.SeederEcUndefined((int)hr); } } throw new FailedToOpenLogTruncContextException(this.m_config.SourceMachine, hr, text); } } catch (TimeoutException ex) { throw new FailedToOpenLogTruncContextException(this.m_config.SourceMachine, 258U, ReplayStrings.ReplayTestStoreConnectivityTimedoutException("LogShipOpenContext", ex.Message), ex); } catch (OperationAbortedException ex2) { throw new FailedToOpenLogTruncContextException(this.m_config.SourceMachine, 1003U, ex2.Message, ex2); } }
private void SetBrokenInternal(FailureTag failureTag, ExEventLog.EventTuple setBrokenEventTuple, ExtendedErrorInfo extendedErrorInfo, params string[] setBrokenArgsPlusDb) { int num; string text = setBrokenEventTuple.EventLogToString(out num, setBrokenArgsPlusDb); Exception failureException = extendedErrorInfo.FailureException; int num2 = 0; if (failureException != null) { num2 = failureException.HResult; } ReplayCrimsonEvents.SetBroken.LogPeriodic <Guid, string, string, string, Exception, int>(this.m_databaseName, DiagCore.DefaultEventSuppressionInterval, this.m_guid, this.m_databaseName, text, Environment.StackTrace, failureException, num2); bool flag = false; lock (this.m_instance) { flag = this.IsBroken; this.FailureInfo.SetBroken(setBrokenEventTuple, new LocalizedString(text), extendedErrorInfo); if (this.m_perfmonCounters != null) { this.m_perfmonCounters.Failed = 1L; this.m_perfmonCounters.Disconnected = 0L; if (this.Suspended) { this.m_perfmonCounters.FailedSuspended = 1L; } } bool flag3; setBrokenEventTuple.LogEvent(this.m_identity, out flag3, setBrokenArgsPlusDb); ExTraceGlobals.ReplicaInstanceTracer.TraceDebug <string, FailureTag, string>((long)this.GetHashCode(), "{0} SetBroken with tag {1} because {2}", this.m_displayName, failureTag, text); MonitoredDatabase monitoredDatabase = MonitoredDatabase.FindMonitoredDatabase(this.ReplicaInstance.Configuration.ServerName, this.m_guid); if (monitoredDatabase != null && this.PassiveSeedingSourceContext != PassiveSeedingSourceContextEnum.None) { ExTraceGlobals.ReplicaInstanceTracer.TraceDebug <Guid>((long)this.GetHashCode(), "Cancel seeding for mdb {0}", this.m_guid); SourceSeedTable.Instance.CancelSeedingIfAppropriate(SourceSeedTable.CancelReason.CopyFailed, monitoredDatabase.DatabaseGuid); } bool flag4 = false; if (flag3 && (!RegistryParameters.DisableSetBrokenFailureItemSuppression || this.IsSuppressableFailureTag(failureTag)) && !this.IsNonSuppressableFailureTag(failureTag)) { flag4 = true; } if (!flag4 && failureTag != FailureTag.NoOp) { FailureItemPublisherHelper.PublishAction(failureTag, this.m_guid, this.m_databaseName); } if (!flag) { this.FailureInfo.PersistFailure(this.m_replayState); } this.ExternalStatus.Refresh(); } if (!flag) { this.m_instance.PrepareToStop(); } }
public void ProcessSourceLogCorruption(long badGeneration, Exception ex) { if (Monitor.TryEnter(this.m_sourceLogCorruptionLock)) { try { long num = this.EndOfLogGeneration - badGeneration; if (num >= (long)RegistryParameters.CorruptLogRequiredRange) { if (badGeneration > this.m_mostRecentSourceLogCorruptionGen) { this.m_mostRecentSourceLogCorruptionGen = badGeneration; FailureItemPublisherHelper.PublishAction(FailureTag.SourceLogCorruptionOutsideRequiredRange, this.DatabaseGuid, this.DatabaseName, null); } } else { ReplicaInstanceManager replicaInstanceManager = Dependencies.ReplayCoreManager.ReplicaInstanceManager; ReplicaInstance replicaInstance = null; if (replicaInstanceManager.TryGetReplicaInstance(this.DatabaseGuid, out replicaInstance) && replicaInstance.CurrentContext.IsBroken) { MonitoredDatabase.Tracer.TraceError((long)this.GetHashCode(), "ProcessSourceLogCorruption ignores error since copy is already broken"); } else { this.m_mostRecentSourceLogCorruptionGen = badGeneration; this.DismountForSourceLogCorruption(ex); FailureItemPublisherHelper.PublishAction(FailureTag.SourceLogCorruption, this.DatabaseGuid, this.DatabaseName, null); } } } finally { Monitor.Exit(this.m_sourceLogCorruptionLock); } } }
protected override void TimerCallbackInternal() { LogTruncater.Tracer.TraceDebug <string, DateTime>((long)this.GetHashCode(), "TimerCallback: {0}: Entered at {1}", this.Configuration.Name, DateTime.UtcNow); if (base.PrepareToStopCalled || this.m_replicaInstanceContext.Initializing || this.m_replicaInstanceContext.Resynchronizing || this.m_replicaInstanceContext.Seeding) { LogTruncater.Tracer.TraceDebug <string>((long)this.GetHashCode(), "TimerCallback: {0}: The instance is stopping or checking stage, abort this timercallback", this.Configuration.Name); return; } if (this.Configuration.DestinationEdbPath == null) { LogTruncater.Tracer.TraceDebug <string>((long)this.GetHashCode(), "TimerCallback: {0}: Returns since we have no database.", this.Configuration.Name); return; } LogTruncater.Tracer.TraceDebug((long)this.GetHashCode(), "TimerCallback: {0} : m_localReplayerGen = 0x{1:X}, m_localInspectorGen = 0x{2:X}, m_lowestGenRequiredGlobally = 0x{3:X}, m_genTruncatedLocally = 0x{4:X}.", new object[] { this.Configuration.Name, this.m_localReplayerGen, this.m_localInspectorGen, this.m_lowestGenRequiredGlobally, this.m_genTruncatedLocally }); long num = this.CalculateGlobalTruncateInput(); if (num <= 0L) { LogTruncater.Tracer.TraceDebug <string>((long)this.GetHashCode(), "TimerCallback: {0}: Exiting because inspector/replayer haven't established position yet.", this.Configuration.Name); return; } num = this.GetExtendedGeneration(num); Exception arg = null; bool flag = false; try { this.LogShipCall(num, false); long num2 = this.CalculatePassiveTruncationPoint(); LooseTruncation looseTruncation = new LooseTruncation(); if (looseTruncation.Enabled && looseTruncation.SpaceIsLow(this.m_configuration)) { long num3 = looseTruncation.MinRequiredGen(this.FileChecker, this.Configuration); LogTruncater.Tracer.TraceDebug <long, long>((long)this.GetHashCode(), "LooseTruncation MinRequiredGen={0} NormalLocalGen={1}", num3, num2); if (num3 > num2) { ReplayCrimsonEvents.LooseTruncationOnPassiveBeingUsed.LogPeriodic <string, string, long, long>(this.Configuration.Identity, DiagCore.DefaultEventSuppressionInterval, this.Configuration.Identity, this.Configuration.DatabaseName, num2, num3); num2 = num3; } } if (num2 > this.m_genTruncatedLocally) { long num4 = num2; if (this.m_configuration.TruncationLagTime != EnhancedTimeSpan.Zero) { if (this.m_genTruncatedLocally == 0L) { num4 = this.FileChecker.FileState.LowestGenerationPresent; } else { num4 = this.m_genTruncatedLocally; } while (num4 < num2) { if (base.PrepareToStopCalled) { throw new OperationAbortedException(); } DateTime fileTime = this.GetFileTime(num4); if ((DateTime)ExDateTime.UtcNow - fileTime < this.m_configuration.TruncationLagTime) { LogTruncater.Tracer.TraceDebug <long, DateTime, string>((long)this.GetHashCode(), "TimerCallback: {2}: Log 0x{0:X} with time '{1}' protected by TruncationLag", num4, fileTime, this.Configuration.Name); break; } num4 += 1L; } } if (num4 != this.m_genTruncatedLocally) { for (long num5 = this.FileChecker.FileState.LowestGenerationPresent; num5 < num4; num5 += 1L) { string text = this.Configuration.BuildFullLogfileName(num5); Exception ex = FileCleanup.Delete(text); if (ex != null) { LogTruncater.Tracer.TraceError <string, Exception>((long)this.GetHashCode(), "Truncation failed to delete {0}: {1}", text, ex); uint hrforException = (uint)Marshal.GetHRForException(ex); throw new FailedToTruncateLocallyException(hrforException, ex.Message, ex); } if (base.PrepareToStopCalled) { throw new OperationAbortedException(); } } this.m_genTruncatedLocally = num4; this.ProbeForLowestLogPresent(); } } flag = true; } catch (OperationAbortedException arg2) { flag = true; LogTruncater.Tracer.TraceDebug <OperationAbortedException>((long)this.GetHashCode(), "Stop detected: {0}", arg2); } catch (IOException ex2) { flag = true; LogTruncater.Tracer.TraceError <string, IOException>((long)this.GetHashCode(), "TimerCallback: {0}: Got an IO exception and probably the file we are going to delete doesn't exist : {1}", this.Configuration.Name, ex2); ReplayEventLogConstants.Tuple_LogTruncationLocalFailure.LogEvent(null, new object[] { this.Configuration.DisplayName, ex2.Message }); } catch (FailedToOpenLogTruncContextException ex3) { arg = ex3; ReplayEventLogConstants.Tuple_LogTruncationOpenFailed.LogEvent(this.Configuration.Identity, new object[] { this.Configuration.DisplayName, ex3.Hresult, ex3.Message }); } catch (CopyUnknownToActiveLogTruncationException ex4) { arg = ex4; ReplayEventLogConstants.Tuple_LogTruncationOpenFailed.LogEvent(this.Configuration.Identity, new object[] { this.Configuration.DisplayName, ex4.Hresult, ex4.Message }); } catch (FailedToNotifySourceLogTruncException ex5) { arg = ex5; ReplayEventLogConstants.Tuple_LogTruncationSourceFailure.LogEvent(null, new object[] { this.Configuration.DisplayName, ex5.Message }); } catch (FailedToTruncateLocallyException ex6) { arg = ex6; ReplayEventLogConstants.Tuple_LogTruncationLocalFailure.LogEvent(this.Configuration.DisplayName, new object[] { this.Configuration.DisplayName, ex6.Message }); FailureTag failureTag = FailureTag.NoOp; uint hresult = ex6.Hresult; if (hresult == 3355444222U) { failureTag = FailureTag.IoHard; } if (failureTag != FailureTag.NoOp) { FailureItemPublisherHelper.PublishAction(failureTag, this.m_configuration.IdentityGuid, this.m_configuration.DatabaseName); } } finally { if (!flag) { LogTruncater.Tracer.TraceError <string, Exception>((long)this.GetHashCode(), "TimerCallback: {0}: Esebcli2 call failed, or unhandled exception occurred. Disposing m_logShipContext... Exception: {1}", this.Configuration.Name, arg); if (this.m_logShipContext != null) { this.m_logShipContext.Dispose(); this.m_logShipContext = null; } LogTruncater.Tracer.TraceDebug <string>((long)this.GetHashCode(), "TimerCallback: {0}: m_logShipContext disposed.", this.Configuration.Name); } } this.m_perfmonCounters.TruncatedGenerationNumber = this.m_genTruncatedLocally; }
public MonitoredDatabaseInitException StartMonitoring() { Exception ex = null; MonitoredDatabase.Tracer.TraceDebug <string, Guid>((long)this.GetHashCode(), "StartMonitoring: '{0}' {1}", this.DatabaseName, this.DatabaseGuid); try { if (!this.m_fPassive) { this.SetupWatcher(); } ExTraceGlobals.FaultInjectionTracer.TraceTest(2294689085U); TimeSpan invokeTimeout = TimeSpan.FromSeconds((double)RegistryParameters.StartupLogScanTimeoutInSec); InvokeWithTimeout.Invoke(delegate() { this.DetermineEndOfLog(); }, invokeTimeout); if (!this.m_fPassive) { this.m_selfCheckEnabled = true; } MonitoredDatabase.Tracer.TraceDebug <string, long>((long)this.GetHashCode(), "Monitoring database {0}. EOL=0x{1:x}", this.m_config.DatabaseName, this.m_currentEndOfLog.Generation); } catch (TimeoutException ex2) { ex = ex2; } catch (ArgumentException ex3) { ex = ex3; } catch (IOException ex4) { ex = ex4; } catch (UnauthorizedAccessException ex5) { ex = ex5; } catch (EsentErrorException ex6) { ex = ex6; } if (ex != null) { string text = string.Format("0x{0:X}", ex.HResult); MonitoredDatabase.Tracer.TraceError <string, string, Exception>((long)this.GetHashCode(), "StartMonitoring database {0} failed with HRESULT {1} exception {2}", this.m_config.DatabaseName, text, ex); ReplayCrimsonEvents.MonitoredDBFailedToStart.LogPeriodic <Guid, string, string, string, string, string>(this.DatabaseName, DiagCore.DefaultEventSuppressionInterval, this.DatabaseGuid, this.DatabaseName, ex.Message, text, ex.ToString(), this.m_sourceDir); if (this.IsPassiveCopy) { ReplayEventLogConstants.Tuple_PassiveMonitoredDBFailedToStart.LogEvent(this.Identity, new object[] { this.DatabaseName, ex.ToString() }); } else { FailureTag failureTag = FailureTag.MonitoredDatabaseFailed; IOException ex7 = ex as IOException; if (ex7 != null && DirectoryOperations.IsPathOnLockedVolume(this.m_sourceDir)) { failureTag = FailureTag.LockedVolume; } ReplayEventLogConstants.Tuple_ActiveMonitoredDBFailedToStart.LogEvent(this.Identity, new object[] { this.DatabaseName, ex.ToString() }); FailureItemPublisherHelper.PublishAction(failureTag, this.DatabaseGuid, this.DatabaseName); } this.StopMonitoring(); return(new MonitoredDatabaseInitException(this.DatabaseName, ex.Message, ex)); } return(null); }