private void RunInternal(TimeSpan waitDuration)
{
GroupStatusInfo groupStatusInfo = this.GroupStatusInfo;
groupStatusInfo.TotalRequested = this.groupConfig.Members.Length;
InstanceGroupMemberConfig[] members = this.groupConfig.Members;
for (int i = 0; i < members.Length; i++)
{
InstanceGroupMemberConfig instanceGroupMemberConfig = members[i];
string memberName = instanceGroupMemberConfig.Name;
groupStatusInfo.StatusMap[memberName] = null;
Task.Factory.StartNew(delegate()
{
this.CollectStatusForServer(memberName, waitDuration);
});
}
try
{
if (this.completionEvent.WaitOne(waitDuration) && groupStatusInfo.TotalNoReplies > 0)
{
TimeSpan timeSpan = DateTimeOffset.Now - groupStatusInfo.CollectionStartTime + waitDuration;
if (timeSpan.TotalMilliseconds > 0.0)
{
int millisecondsTimeout = Math.Min((int)timeSpan.TotalMilliseconds, 2000);
Thread.Sleep(millisecondsTimeout);
}
}
}
finally
{
this.MarkCompletion();
}
}
public void Initialize(string self, InstanceClientFactory instanceClientFactory, InstanceGroupConfig groupConfig, IDxStoreEventLogger eventLogger, double requiredSuccessPercent)
{
this.EventLogger = eventLogger;
this.self = self;
this.instanceClientFactory = instanceClientFactory;
this.GroupStatusInfo = new GroupStatusInfo();
this.groupConfig = groupConfig;
this.completionEvent = new ManualResetEvent(false);
this.requiredSuccessPercent = requiredSuccessPercent;
}
private void DetermineStoreState(GroupStatusInfo gsi)
{
int lag = gsi.Lag;
if (gsi.LocalInstance == null)
{
this.StoreState = StoreState.Initializing;
return;
}
StoreState storeState;
if (lag > 0)
{
bool flag = false;
GroupStatusInfo groupStatusInfo = this.groupStatuses.LastOrDefault <GroupStatusInfo>();
if (groupStatusInfo != null && groupStatusInfo.Lag > 0 && groupStatusInfo.LocalInstance != null && groupStatusInfo.LocalInstance.InstanceNumber == gsi.LocalInstance.InstanceNumber)
{
flag = true;
}
if (!flag)
{
if (lag <= this.instance.GroupConfig.Settings.MaximumAllowedInstanceNumberLag)
{
storeState = (StoreState.Current | StoreState.CatchingUp);
}
else
{
storeState = (StoreState.Stale | StoreState.CatchingUp);
}
}
else
{
storeState = (StoreState.Stale | StoreState.Struck);
}
}
else
{
storeState = StoreState.Current;
}
if (gsi.IsMajoritySuccessfulyReplied)
{
this.ConsecutiveNoMajority = 0;
this.ConsecutiveMajority++;
storeState &= ~StoreState.NoMajority;
}
else
{
storeState |= StoreState.NoMajority;
this.ConsecutiveMajority = 0;
this.ConsecutiveNoMajority++;
}
this.StoreState = storeState;
}
private void UpdateStatusInternal(string serverName, InstanceStatusInfo statusInfo, Exception ex)
{
GroupStatusInfo groupStatusInfo = this.GroupStatusInfo;
if (!this.isCompleted)
{
groupStatusInfo.StatusMap[serverName] = statusInfo;
if (ex == null)
{
groupStatusInfo.TotalSuccessful++;
}
else
{
groupStatusInfo.TotalFailed++;
}
if (!this.isSignaled)
{
if (this.completionEvent != null)
{
double num = (double)groupStatusInfo.TotalSuccessful * 100.0 / (double)groupStatusInfo.TotalRequested;
if (groupStatusInfo.TotalRequested == groupStatusInfo.TotalSuccessful + groupStatusInfo.TotalFailed || num >= this.requiredSuccessPercent)
{
GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Signaling completion (Total: {1}, Success: {2}, Failed: {3}", new object[]
{
this.groupConfig.Identity,
groupStatusInfo.TotalRequested,
groupStatusInfo.TotalSuccessful,
groupStatusInfo.TotalFailed
});
this.completionEvent.Set();
this.isSignaled = true;
return;
}
}
}
else
{
GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Already signalled but trying level best to receive max replies in the grace time (Total: {1}, Success: {2}, Failed: {3}", new object[]
{
this.groupConfig.Identity,
groupStatusInfo.TotalRequested,
groupStatusInfo.TotalSuccessful,
groupStatusInfo.TotalFailed
});
}
}
}
private void WhenHealthCheckerSeeMajorityOfNodes(GroupStatusInfo gsi)
{
if (this.IsStartupCompleted)
{
return;
}
bool flag = false;
try
{
object obj;
Monitor.Enter(obj = this.instanceLock, ref flag);
bool isWaitForNextRound = false;
bool flag2 = gsi.Lag > this.GroupConfig.Settings.MaxAllowedLagToCatchup;
DxStoreInstance.Tracer.TraceDebug((long)this.IdentityHash, "{0}: Instance start - Majority replied (LocalInstance# {1}, Lag: {2}, CatchupLimit: {3}", new object[]
{
this.Identity,
(gsi.LocalInstance != null) ? gsi.LocalInstance.InstanceNumber : -1,
gsi.Lag,
this.GroupConfig.Settings.MaxAllowedLagToCatchup
});
if (flag2)
{
isWaitForNextRound = true;
}
DxStoreInstanceClient client = this.InstanceClientFactory.GetClient(gsi.HighestInstance.NodeName);
InstanceSnapshotInfo snapshotInfo = null;
this.RunBestEffortOperation("GetSnapshot :" + gsi.HighestInstance.NodeName, delegate
{
snapshotInfo = client.AcquireSnapshot(null, true, null);
}, LogOptions.LogAll, null, null, null, null);
if (snapshotInfo != null && snapshotInfo.LastInstanceExecuted > gsi.LocalInstance.InstanceNumber)
{
this.RunBestEffortOperation("Apply local snapshot", delegate
{
this.SnapshotManager.ApplySnapshot(snapshotInfo, true);
isWaitForNextRound = false;
}, LogOptions.LogAll, null, null, null, null);
}
if (!isWaitForNextRound)
{
this.majorityNotificationSubscription.Dispose();
this.majorityNotificationSubscription = null;
this.HealthChecker.ChangeTimerDuration(this.GroupConfig.Settings.GroupHealthCheckDuration);
Round <string>?leaderHint = null;
if (gsi.IsLeaderExist)
{
leaderHint = new Round <string>?(gsi.LeaderHint);
}
this.StateMachine = this.CreateStateMachine(leaderHint, null);
Task task = this.StartStateMachine();
task.ContinueWith(delegate(Task t)
{
this.EventLogger.Log(DxEventSeverity.Info, 0, "Successfully started state machine", new object[0]);
this.SnapshotManager.Start();
this.State = InstanceState.Running;
this.IsStartupCompleted = true;
});
}
else
{
DxStoreInstance.Tracer.TraceWarning <string>((long)this.IdentityHash, "{0}: Instance start - waiting for the next round to start local instance", this.Identity);
}
}
finally
{
if (flag)
{
object obj;
Monitor.Exit(obj);
}
}
}