private void UpdateStatus(string serverName, InstanceStatusInfo statusInfo, Exception ex)
{
lock (this.locker)
{
this.UpdateStatusInternal(serverName, statusInfo, ex);
}
}
public InstanceStatusInfo GetStatus()
{
if (this.SnapshotManager == null || !this.SnapshotManager.IsInitialLoadAttempted)
{
throw new DxStoreInstanceNotReadyException(this.State.ToString());
}
InstanceStatusInfo instanceStatusInfo = new InstanceStatusInfo();
instanceStatusInfo.State = this.State;
instanceStatusInfo.Self = this.GroupConfig.Self;
instanceStatusInfo.MemberConfigs = this.GroupConfig.Members;
if (this.LocalDataStore != null)
{
instanceStatusInfo.LastInstanceExecuted = this.LocalDataStore.LastInstanceExecuted;
}
if (this.StateMachine != null)
{
instanceStatusInfo.PaxosInfo = this.StateMachine.GetPaxosInfo();
}
instanceStatusInfo.HostProcessInfo = this.LocalProcessInfo;
if (this.CommitAcknowledger != null)
{
instanceStatusInfo.CommitAckOldestItemTime = this.CommitAcknowledger.OldestItemTime;
}
return(instanceStatusInfo);
}
private void CollectStatusForServer(string serverName, TimeSpan waitDuration)
{
InstanceStatusInfo statusInfo = null;
Exception ex = null;
string label = string.Format("GetStatus({0})", serverName);
if (!this.groupConfig.Settings.IsUseHttpTransportForInstanceCommunication)
{
ex = Utils.RunOperation(this.groupConfig.Identity, label, delegate
{
DxStoreInstanceClient client = this.instanceClientFactory.GetClient(serverName);
statusInfo = client.GetStatus(null);
}, this.EventLogger, LogOptions.LogException | this.groupConfig.Settings.AdditionalLogOptions, true, null, new TimeSpan?(this.groupConfig.Settings.PeriodicExceptionLoggingDuration), null, null, null);
}
else
{
try
{
string memberNetworkAddress = this.groupConfig.GetMemberNetworkAddress(serverName);
Task <InstanceStatusInfo> statusAsync = HttpClient.GetStatusAsync(memberNetworkAddress, serverName, this.groupConfig.Name, this.self);
statusAsync.Wait(waitDuration);
statusInfo = statusAsync.Result;
}
catch (Exception ex2)
{
ex = ex2;
this.EventLogger.Log(DxEventSeverity.Error, 0, "http send for GetStatusAsync failed: {0}", new object[]
{
ex2.ToString()
});
}
}
this.UpdateStatus(serverName, statusInfo, ex);
}
public string[] GetPaxosMembers(string nodeName)
{
InstanceStatusInfo instanceStatusInfo = this.StatusMap.Values.FirstOrDefault((InstanceStatusInfo v) => v != null && Utils.IsEqual(v.Self, nodeName, StringComparison.OrdinalIgnoreCase));
if (instanceStatusInfo != null && instanceStatusInfo.PaxosInfo != null)
{
return(instanceStatusInfo.PaxosInfo.Members);
}
return(null);
}
public InstanceStatusInfo GetMemberInfo(string memberName)
{
InstanceStatusInfo result = null;
if (this.StatusMap != null)
{
this.StatusMap.TryGetValue(memberName, out result);
}
return(result);
}
private void CheckHealthInternal()
{
bool flag = true;
bool flag2 = false;
InstanceStatusInfo instanceStatus = this.GetInstanceStatus();
if (instanceStatus != null)
{
if (instanceStatus.State == InstanceState.Running || instanceStatus.State == InstanceState.Starting)
{
this.firstObservedUnhealthyTime = DateTimeOffset.MinValue;
flag = false;
}
}
else if (this.processInfo == null)
{
flag2 = true;
}
if (flag)
{
if (this.firstObservedUnhealthyTime == DateTimeOffset.MinValue)
{
this.firstObservedUnhealthyTime = DateTimeOffset.Now;
}
if (DateTimeOffset.Now - this.firstObservedUnhealthyTime >= this.config.Settings.DurationToWaitBeforeRestart)
{
flag2 = true;
}
this.manager.EventLogger.Log(DxEventSeverity.Warning, 0, "{0}: Instance is unhealthy. (IsRestartRequired: {1}, FirstUnhealthyTime: {2})", new object[]
{
this.config.Identity,
flag2,
this.firstObservedUnhealthyTime
});
}
else if (this.config.Settings.IsAllowDynamicReconfig && instanceStatus != null && instanceStatus.IsValidPaxosMembersExist() && instanceStatus.IsValidLeaderExist() && !instanceStatus.AreMembersSame(this.config.Members))
{
bool flag4;
bool flag5;
bool flag3 = this.ReconfigureBestEffort("Checker.ReconfigureBestEffort", this.config.Members, instanceStatus, out flag4, out flag5);
if (flag4 && !flag3)
{
flag2 = true;
}
}
if (flag2)
{
this.IsRestartRequested = true;
Task.Factory.StartNew(delegate()
{
this.manager.RestartInstance(this.config.Name, false);
});
}
}
private bool StartProcess(InstanceStatusInfo statusInfo)
{
DxStoreInstanceChecker.< > c__DisplayClass19 CS$ < > 8__locals1 = new DxStoreInstanceChecker.< > c__DisplayClass19();
CS$ < > 8__locals1.statusInfo = statusInfo;
CS$ < > 8__locals1.< > 4__this = this;
CS$ < > 8__locals1.cmdLineArguments = string.Format("-GroupName:{0} -Self:{1}", this.config.Name, this.config.Self);
if (this.config.IsZeroboxMode)
{
DxStoreInstanceChecker.< > c__DisplayClass19 CS$ < > 8__locals2 = CS$ < > 8__locals1;
CS$ < > 8__locals2.cmdLineArguments += " -ZeroboxMode";
}
Utils.RunOperation(this.config.Identity, string.Format("Starting Process : {0} {1}", this.config.Settings.InstanceProcessName, CS$ < > 8__locals1.cmdLineArguments), delegate
{
CS$ < > 8__locals1.< > 4__this.StartProcessInternal(CS$ < > 8__locals1.statusInfo, CS$ < > 8__locals1.cmdLineArguments);
}, this.manager.EventLogger, LogOptions.LogAll | this.config.Settings.AdditionalLogOptions, false, null, null, null, null, null);
// Token: 0x06000EB4 RID: 3764 RVA: 0x0003ED7C File Offset: 0x0003CF7C
public bool IsPaxosConfiguredAndLeaderExist(out string statusInfoStr)
{
statusInfoStr = "<null>";
this.InitializeIfRequired();
InstanceStatusInfo si = null;
Utils.RunBestEffort(delegate
{
si = this.ClientFactory.LocalClient.GetStatus(null);
});
if (si != null)
{
statusInfoStr = Utils.SerializeObjectToJsonString <InstanceStatusInfo>(si, false, true);
}
return(si != null && si.IsValidLeaderExist());
}
private void UpdateStatusInternal(string serverName, InstanceStatusInfo statusInfo, Exception ex)
{
GroupStatusInfo groupStatusInfo = this.GroupStatusInfo;
if (!this.isCompleted)
{
groupStatusInfo.StatusMap[serverName] = statusInfo;
if (ex == null)
{
groupStatusInfo.TotalSuccessful++;
}
else
{
groupStatusInfo.TotalFailed++;
}
if (!this.isSignaled)
{
if (this.completionEvent != null)
{
double num = (double)groupStatusInfo.TotalSuccessful * 100.0 / (double)groupStatusInfo.TotalRequested;
if (groupStatusInfo.TotalRequested == groupStatusInfo.TotalSuccessful + groupStatusInfo.TotalFailed || num >= this.requiredSuccessPercent)
{
GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Signaling completion (Total: {1}, Success: {2}, Failed: {3}", new object[]
{
this.groupConfig.Identity,
groupStatusInfo.TotalRequested,
groupStatusInfo.TotalSuccessful,
groupStatusInfo.TotalFailed
});
this.completionEvent.Set();
this.isSignaled = true;
return;
}
}
}
else
{
GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Already signalled but trying level best to receive max replies in the grace time (Total: {1}, Success: {2}, Failed: {3}", new object[]
{
this.groupConfig.Identity,
groupStatusInfo.TotalRequested,
groupStatusInfo.TotalSuccessful,
groupStatusInfo.TotalFailed
});
}
}
}
private void StartInstance()
{
InstanceStatusInfo status = this.GetInstanceStatus();
Exception ex = Utils.RunWithPrivilege("SeDebugPrivilege", delegate
{
this.StartProcess(status);
});
if (ex != null)
{
this.manager.EventLogger.Log(DxEventSeverity.Error, 0, "{0}: StartProcess failed with privilege exception {1}", new object[]
{
this.config.Identity,
ex
});
}
}
public PaxosBasicInfo GetBestPossiblePaxosConfig()
{
InstanceStatusInfo instanceStatusInfo = (from s in this.StatusMap.Values
where s != null && s.PaxosInfo != null
select s).OrderByDescending(delegate(InstanceStatusInfo s)
{
if (!s.IsLeader)
{
return(0);
}
return(1);
}).ThenByDescending((InstanceStatusInfo s) => s.LastInstanceExecuted).FirstOrDefault <InstanceStatusInfo>();
if (instanceStatusInfo != null)
{
return(instanceStatusInfo.PaxosInfo);
}
return(null);
}
private InstanceStatusInfo GetInstanceStatus()
{
InstanceStatusInfo statusInfo = null;
Exception ex = Utils.RunOperation(this.config.Identity, "GetInstanceStatus", delegate
{
statusInfo = this.InstanceClientFactory.LocalClient.GetStatus(null);
}, this.manager.EventLogger, LogOptions.LogException | LogOptions.LogPeriodic | this.config.Settings.AdditionalLogOptions, true, null, null, null, null, null);
if (ex == null)
{
if (DxStoreManager.Tracer.IsTraceEnabled(TraceType.DebugTrace))
{
string arg = Utils.SerializeObjectToJsonString <InstanceStatusInfo>(statusInfo, false, true) ?? "<serialization error>";
DxStoreManager.Tracer.TraceDebug <string, string>((long)this.config.Identity.GetHashCode(), "{0}: InstanceChecker.GetInstanceStatus() returned {1}", this.config.Identity, arg);
}
}
else if (DxStoreManager.Tracer.IsTraceEnabled(TraceType.ErrorTrace))
{
DxStoreManager.Tracer.TraceError <string, Exception>((long)this.config.Identity.GetHashCode(), "{0}: InstanceChecker.GetInstanceStatus() failed with {1}", this.config.Identity, ex);
}
return(statusInfo);
}
public bool ReconfigureBestEffort(string callerLabel, InstanceGroupMemberConfig[] members, InstanceStatusInfo si, out bool isReconfiguredAttempted, out bool isPaxosMembershipInSync)
{
bool isSuccess = false;
isReconfiguredAttempted = false;
isPaxosMembershipInSync = false;
lock (this.locker)
{
if (si != null && si.IsLeader)
{
if (this.config.Settings.IsAppendOnlyMembership)
{
Dictionary <string, InstanceGroupMemberConfig> dictionary = members.ToDictionary((InstanceGroupMemberConfig m) => m.Name);
foreach (string text in si.PaxosInfo.Members)
{
InstanceGroupMemberConfig value = null;
if (!dictionary.TryGetValue(text, out value))
{
value = new InstanceGroupMemberConfig
{
Name = text
};
dictionary.Add(text, value);
}
}
members = dictionary.Values.ToArray <InstanceGroupMemberConfig>();
}
isPaxosMembershipInSync = si.AreMembersSame(members);
if (!isPaxosMembershipInSync)
{
isReconfiguredAttempted = true;
string arg = string.Join(",", si.PaxosInfo.Members.ToArray <string>());
string arg2 = string.Join(",", (from m in members
select m.Name).ToArray <string>());
string context = string.Format("Current Paxos Members: '{0}' - New Membership attempted: '{1}'", arg, arg2);
Utils.RunOperation(this.config.Identity, callerLabel, delegate
{
this.InstanceClientFactory.LocalClient.Reconfigure(members, null);
isSuccess = true;
}, this.manager.EventLogger, LogOptions.LogAll | this.config.Settings.AdditionalLogOptions, true, new TimeSpan?(this.config.Settings.MemberReconfigureTimeout), null, null, null, context);
}
}
}
return(isSuccess);
}
