private void UpdateStatus(string serverName, InstanceStatusInfo statusInfo, Exception ex) { lock (this.locker) { this.UpdateStatusInternal(serverName, statusInfo, ex); } }
public InstanceStatusInfo GetStatus() { if (this.SnapshotManager == null || !this.SnapshotManager.IsInitialLoadAttempted) { throw new DxStoreInstanceNotReadyException(this.State.ToString()); } InstanceStatusInfo instanceStatusInfo = new InstanceStatusInfo(); instanceStatusInfo.State = this.State; instanceStatusInfo.Self = this.GroupConfig.Self; instanceStatusInfo.MemberConfigs = this.GroupConfig.Members; if (this.LocalDataStore != null) { instanceStatusInfo.LastInstanceExecuted = this.LocalDataStore.LastInstanceExecuted; } if (this.StateMachine != null) { instanceStatusInfo.PaxosInfo = this.StateMachine.GetPaxosInfo(); } instanceStatusInfo.HostProcessInfo = this.LocalProcessInfo; if (this.CommitAcknowledger != null) { instanceStatusInfo.CommitAckOldestItemTime = this.CommitAcknowledger.OldestItemTime; } return(instanceStatusInfo); }
private void CollectStatusForServer(string serverName, TimeSpan waitDuration) { InstanceStatusInfo statusInfo = null; Exception ex = null; string label = string.Format("GetStatus({0})", serverName); if (!this.groupConfig.Settings.IsUseHttpTransportForInstanceCommunication) { ex = Utils.RunOperation(this.groupConfig.Identity, label, delegate { DxStoreInstanceClient client = this.instanceClientFactory.GetClient(serverName); statusInfo = client.GetStatus(null); }, this.EventLogger, LogOptions.LogException | this.groupConfig.Settings.AdditionalLogOptions, true, null, new TimeSpan?(this.groupConfig.Settings.PeriodicExceptionLoggingDuration), null, null, null); } else { try { string memberNetworkAddress = this.groupConfig.GetMemberNetworkAddress(serverName); Task <InstanceStatusInfo> statusAsync = HttpClient.GetStatusAsync(memberNetworkAddress, serverName, this.groupConfig.Name, this.self); statusAsync.Wait(waitDuration); statusInfo = statusAsync.Result; } catch (Exception ex2) { ex = ex2; this.EventLogger.Log(DxEventSeverity.Error, 0, "http send for GetStatusAsync failed: {0}", new object[] { ex2.ToString() }); } } this.UpdateStatus(serverName, statusInfo, ex); }
public string[] GetPaxosMembers(string nodeName) { InstanceStatusInfo instanceStatusInfo = this.StatusMap.Values.FirstOrDefault((InstanceStatusInfo v) => v != null && Utils.IsEqual(v.Self, nodeName, StringComparison.OrdinalIgnoreCase)); if (instanceStatusInfo != null && instanceStatusInfo.PaxosInfo != null) { return(instanceStatusInfo.PaxosInfo.Members); } return(null); }
public InstanceStatusInfo GetMemberInfo(string memberName) { InstanceStatusInfo result = null; if (this.StatusMap != null) { this.StatusMap.TryGetValue(memberName, out result); } return(result); }
private void CheckHealthInternal() { bool flag = true; bool flag2 = false; InstanceStatusInfo instanceStatus = this.GetInstanceStatus(); if (instanceStatus != null) { if (instanceStatus.State == InstanceState.Running || instanceStatus.State == InstanceState.Starting) { this.firstObservedUnhealthyTime = DateTimeOffset.MinValue; flag = false; } } else if (this.processInfo == null) { flag2 = true; } if (flag) { if (this.firstObservedUnhealthyTime == DateTimeOffset.MinValue) { this.firstObservedUnhealthyTime = DateTimeOffset.Now; } if (DateTimeOffset.Now - this.firstObservedUnhealthyTime >= this.config.Settings.DurationToWaitBeforeRestart) { flag2 = true; } this.manager.EventLogger.Log(DxEventSeverity.Warning, 0, "{0}: Instance is unhealthy. (IsRestartRequired: {1}, FirstUnhealthyTime: {2})", new object[] { this.config.Identity, flag2, this.firstObservedUnhealthyTime }); } else if (this.config.Settings.IsAllowDynamicReconfig && instanceStatus != null && instanceStatus.IsValidPaxosMembersExist() && instanceStatus.IsValidLeaderExist() && !instanceStatus.AreMembersSame(this.config.Members)) { bool flag4; bool flag5; bool flag3 = this.ReconfigureBestEffort("Checker.ReconfigureBestEffort", this.config.Members, instanceStatus, out flag4, out flag5); if (flag4 && !flag3) { flag2 = true; } } if (flag2) { this.IsRestartRequested = true; Task.Factory.StartNew(delegate() { this.manager.RestartInstance(this.config.Name, false); }); } }
private bool StartProcess(InstanceStatusInfo statusInfo) { DxStoreInstanceChecker.< > c__DisplayClass19 CS$ < > 8__locals1 = new DxStoreInstanceChecker.< > c__DisplayClass19(); CS$ < > 8__locals1.statusInfo = statusInfo; CS$ < > 8__locals1.< > 4__this = this; CS$ < > 8__locals1.cmdLineArguments = string.Format("-GroupName:{0} -Self:{1}", this.config.Name, this.config.Self); if (this.config.IsZeroboxMode) { DxStoreInstanceChecker.< > c__DisplayClass19 CS$ < > 8__locals2 = CS$ < > 8__locals1; CS$ < > 8__locals2.cmdLineArguments += " -ZeroboxMode"; } Utils.RunOperation(this.config.Identity, string.Format("Starting Process : {0} {1}", this.config.Settings.InstanceProcessName, CS$ < > 8__locals1.cmdLineArguments), delegate { CS$ < > 8__locals1.< > 4__this.StartProcessInternal(CS$ < > 8__locals1.statusInfo, CS$ < > 8__locals1.cmdLineArguments); }, this.manager.EventLogger, LogOptions.LogAll | this.config.Settings.AdditionalLogOptions, false, null, null, null, null, null);
// Token: 0x06000EB4 RID: 3764 RVA: 0x0003ED7C File Offset: 0x0003CF7C public bool IsPaxosConfiguredAndLeaderExist(out string statusInfoStr) { statusInfoStr = "<null>"; this.InitializeIfRequired(); InstanceStatusInfo si = null; Utils.RunBestEffort(delegate { si = this.ClientFactory.LocalClient.GetStatus(null); }); if (si != null) { statusInfoStr = Utils.SerializeObjectToJsonString <InstanceStatusInfo>(si, false, true); } return(si != null && si.IsValidLeaderExist()); }
private void UpdateStatusInternal(string serverName, InstanceStatusInfo statusInfo, Exception ex) { GroupStatusInfo groupStatusInfo = this.GroupStatusInfo; if (!this.isCompleted) { groupStatusInfo.StatusMap[serverName] = statusInfo; if (ex == null) { groupStatusInfo.TotalSuccessful++; } else { groupStatusInfo.TotalFailed++; } if (!this.isSignaled) { if (this.completionEvent != null) { double num = (double)groupStatusInfo.TotalSuccessful * 100.0 / (double)groupStatusInfo.TotalRequested; if (groupStatusInfo.TotalRequested == groupStatusInfo.TotalSuccessful + groupStatusInfo.TotalFailed || num >= this.requiredSuccessPercent) { GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Signaling completion (Total: {1}, Success: {2}, Failed: {3}", new object[] { this.groupConfig.Identity, groupStatusInfo.TotalRequested, groupStatusInfo.TotalSuccessful, groupStatusInfo.TotalFailed }); this.completionEvent.Set(); this.isSignaled = true; return; } } } else { GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Already signalled but trying level best to receive max replies in the grace time (Total: {1}, Success: {2}, Failed: {3}", new object[] { this.groupConfig.Identity, groupStatusInfo.TotalRequested, groupStatusInfo.TotalSuccessful, groupStatusInfo.TotalFailed }); } } }
private void StartInstance() { InstanceStatusInfo status = this.GetInstanceStatus(); Exception ex = Utils.RunWithPrivilege("SeDebugPrivilege", delegate { this.StartProcess(status); }); if (ex != null) { this.manager.EventLogger.Log(DxEventSeverity.Error, 0, "{0}: StartProcess failed with privilege exception {1}", new object[] { this.config.Identity, ex }); } }
public PaxosBasicInfo GetBestPossiblePaxosConfig() { InstanceStatusInfo instanceStatusInfo = (from s in this.StatusMap.Values where s != null && s.PaxosInfo != null select s).OrderByDescending(delegate(InstanceStatusInfo s) { if (!s.IsLeader) { return(0); } return(1); }).ThenByDescending((InstanceStatusInfo s) => s.LastInstanceExecuted).FirstOrDefault <InstanceStatusInfo>(); if (instanceStatusInfo != null) { return(instanceStatusInfo.PaxosInfo); } return(null); }
private InstanceStatusInfo GetInstanceStatus() { InstanceStatusInfo statusInfo = null; Exception ex = Utils.RunOperation(this.config.Identity, "GetInstanceStatus", delegate { statusInfo = this.InstanceClientFactory.LocalClient.GetStatus(null); }, this.manager.EventLogger, LogOptions.LogException | LogOptions.LogPeriodic | this.config.Settings.AdditionalLogOptions, true, null, null, null, null, null); if (ex == null) { if (DxStoreManager.Tracer.IsTraceEnabled(TraceType.DebugTrace)) { string arg = Utils.SerializeObjectToJsonString <InstanceStatusInfo>(statusInfo, false, true) ?? "<serialization error>"; DxStoreManager.Tracer.TraceDebug <string, string>((long)this.config.Identity.GetHashCode(), "{0}: InstanceChecker.GetInstanceStatus() returned {1}", this.config.Identity, arg); } } else if (DxStoreManager.Tracer.IsTraceEnabled(TraceType.ErrorTrace)) { DxStoreManager.Tracer.TraceError <string, Exception>((long)this.config.Identity.GetHashCode(), "{0}: InstanceChecker.GetInstanceStatus() failed with {1}", this.config.Identity, ex); } return(statusInfo); }
public bool ReconfigureBestEffort(string callerLabel, InstanceGroupMemberConfig[] members, InstanceStatusInfo si, out bool isReconfiguredAttempted, out bool isPaxosMembershipInSync) { bool isSuccess = false; isReconfiguredAttempted = false; isPaxosMembershipInSync = false; lock (this.locker) { if (si != null && si.IsLeader) { if (this.config.Settings.IsAppendOnlyMembership) { Dictionary <string, InstanceGroupMemberConfig> dictionary = members.ToDictionary((InstanceGroupMemberConfig m) => m.Name); foreach (string text in si.PaxosInfo.Members) { InstanceGroupMemberConfig value = null; if (!dictionary.TryGetValue(text, out value)) { value = new InstanceGroupMemberConfig { Name = text }; dictionary.Add(text, value); } } members = dictionary.Values.ToArray <InstanceGroupMemberConfig>(); } isPaxosMembershipInSync = si.AreMembersSame(members); if (!isPaxosMembershipInSync) { isReconfiguredAttempted = true; string arg = string.Join(",", si.PaxosInfo.Members.ToArray <string>()); string arg2 = string.Join(",", (from m in members select m.Name).ToArray <string>()); string context = string.Format("Current Paxos Members: '{0}' - New Membership attempted: '{1}'", arg, arg2); Utils.RunOperation(this.config.Identity, callerLabel, delegate { this.InstanceClientFactory.LocalClient.Reconfigure(members, null); isSuccess = true; }, this.manager.EventLogger, LogOptions.LogAll | this.config.Settings.AdditionalLogOptions, true, new TimeSpan?(this.config.Settings.MemberReconfigureTimeout), null, null, null, context); } } } return(isSuccess); }