private void RunInternal(TimeSpan waitDuration)
        {
            GroupStatusInfo groupStatusInfo = this.GroupStatusInfo;

            groupStatusInfo.TotalRequested = this.groupConfig.Members.Length;
            InstanceGroupMemberConfig[] members = this.groupConfig.Members;
            for (int i = 0; i < members.Length; i++)
            {
                InstanceGroupMemberConfig instanceGroupMemberConfig = members[i];
                string memberName = instanceGroupMemberConfig.Name;
                groupStatusInfo.StatusMap[memberName] = null;
                Task.Factory.StartNew(delegate()
                {
                    this.CollectStatusForServer(memberName, waitDuration);
                });
            }
            try
            {
                if (this.completionEvent.WaitOne(waitDuration) && groupStatusInfo.TotalNoReplies > 0)
                {
                    TimeSpan timeSpan = DateTimeOffset.Now - groupStatusInfo.CollectionStartTime + waitDuration;
                    if (timeSpan.TotalMilliseconds > 0.0)
                    {
                        int millisecondsTimeout = Math.Min((int)timeSpan.TotalMilliseconds, 2000);
                        Thread.Sleep(millisecondsTimeout);
                    }
                }
            }
            finally
            {
                this.MarkCompletion();
            }
        }
 public void Initialize(string self, InstanceClientFactory instanceClientFactory, InstanceGroupConfig groupConfig, IDxStoreEventLogger eventLogger, double requiredSuccessPercent)
 {
     this.EventLogger            = eventLogger;
     this.self                   = self;
     this.instanceClientFactory  = instanceClientFactory;
     this.GroupStatusInfo        = new GroupStatusInfo();
     this.groupConfig            = groupConfig;
     this.completionEvent        = new ManualResetEvent(false);
     this.requiredSuccessPercent = requiredSuccessPercent;
 }
예제 #3
0
        private void DetermineStoreState(GroupStatusInfo gsi)
        {
            int lag = gsi.Lag;

            if (gsi.LocalInstance == null)
            {
                this.StoreState = StoreState.Initializing;
                return;
            }
            StoreState storeState;

            if (lag > 0)
            {
                bool            flag            = false;
                GroupStatusInfo groupStatusInfo = this.groupStatuses.LastOrDefault <GroupStatusInfo>();
                if (groupStatusInfo != null && groupStatusInfo.Lag > 0 && groupStatusInfo.LocalInstance != null && groupStatusInfo.LocalInstance.InstanceNumber == gsi.LocalInstance.InstanceNumber)
                {
                    flag = true;
                }
                if (!flag)
                {
                    if (lag <= this.instance.GroupConfig.Settings.MaximumAllowedInstanceNumberLag)
                    {
                        storeState = (StoreState.Current | StoreState.CatchingUp);
                    }
                    else
                    {
                        storeState = (StoreState.Stale | StoreState.CatchingUp);
                    }
                }
                else
                {
                    storeState = (StoreState.Stale | StoreState.Struck);
                }
            }
            else
            {
                storeState = StoreState.Current;
            }
            if (gsi.IsMajoritySuccessfulyReplied)
            {
                this.ConsecutiveNoMajority = 0;
                this.ConsecutiveMajority++;
                storeState &= ~StoreState.NoMajority;
            }
            else
            {
                storeState |= StoreState.NoMajority;
                this.ConsecutiveMajority = 0;
                this.ConsecutiveNoMajority++;
            }
            this.StoreState = storeState;
        }
        private void UpdateStatusInternal(string serverName, InstanceStatusInfo statusInfo, Exception ex)
        {
            GroupStatusInfo groupStatusInfo = this.GroupStatusInfo;

            if (!this.isCompleted)
            {
                groupStatusInfo.StatusMap[serverName] = statusInfo;
                if (ex == null)
                {
                    groupStatusInfo.TotalSuccessful++;
                }
                else
                {
                    groupStatusInfo.TotalFailed++;
                }
                if (!this.isSignaled)
                {
                    if (this.completionEvent != null)
                    {
                        double num = (double)groupStatusInfo.TotalSuccessful * 100.0 / (double)groupStatusInfo.TotalRequested;
                        if (groupStatusInfo.TotalRequested == groupStatusInfo.TotalSuccessful + groupStatusInfo.TotalFailed || num >= this.requiredSuccessPercent)
                        {
                            GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Signaling completion (Total: {1}, Success: {2}, Failed: {3}", new object[]
                            {
                                this.groupConfig.Identity,
                                groupStatusInfo.TotalRequested,
                                groupStatusInfo.TotalSuccessful,
                                groupStatusInfo.TotalFailed
                            });
                            this.completionEvent.Set();
                            this.isSignaled = true;
                            return;
                        }
                    }
                }
                else
                {
                    GroupStatusCollector.Tracer.TraceDebug((long)this.groupConfig.Identity.GetHashCode(), "{0}: Already signalled but trying level best to receive max replies in the grace time (Total: {1}, Success: {2}, Failed: {3}", new object[]
                    {
                        this.groupConfig.Identity,
                        groupStatusInfo.TotalRequested,
                        groupStatusInfo.TotalSuccessful,
                        groupStatusInfo.TotalFailed
                    });
                }
            }
        }
예제 #5
0
        private void WhenHealthCheckerSeeMajorityOfNodes(GroupStatusInfo gsi)
        {
            if (this.IsStartupCompleted)
            {
                return;
            }
            bool flag = false;

            try
            {
                object obj;
                Monitor.Enter(obj = this.instanceLock, ref flag);
                bool isWaitForNextRound = false;
                bool flag2 = gsi.Lag > this.GroupConfig.Settings.MaxAllowedLagToCatchup;
                DxStoreInstance.Tracer.TraceDebug((long)this.IdentityHash, "{0}: Instance start - Majority replied (LocalInstance# {1}, Lag: {2}, CatchupLimit: {3}", new object[]
                {
                    this.Identity,
                    (gsi.LocalInstance != null) ? gsi.LocalInstance.InstanceNumber : -1,
                    gsi.Lag,
                    this.GroupConfig.Settings.MaxAllowedLagToCatchup
                });
                if (flag2)
                {
                    isWaitForNextRound = true;
                }
                DxStoreInstanceClient client       = this.InstanceClientFactory.GetClient(gsi.HighestInstance.NodeName);
                InstanceSnapshotInfo  snapshotInfo = null;
                this.RunBestEffortOperation("GetSnapshot :" + gsi.HighestInstance.NodeName, delegate
                {
                    snapshotInfo = client.AcquireSnapshot(null, true, null);
                }, LogOptions.LogAll, null, null, null, null);
                if (snapshotInfo != null && snapshotInfo.LastInstanceExecuted > gsi.LocalInstance.InstanceNumber)
                {
                    this.RunBestEffortOperation("Apply local snapshot", delegate
                    {
                        this.SnapshotManager.ApplySnapshot(snapshotInfo, true);
                        isWaitForNextRound = false;
                    }, LogOptions.LogAll, null, null, null, null);
                }
                if (!isWaitForNextRound)
                {
                    this.majorityNotificationSubscription.Dispose();
                    this.majorityNotificationSubscription = null;
                    this.HealthChecker.ChangeTimerDuration(this.GroupConfig.Settings.GroupHealthCheckDuration);
                    Round <string>?leaderHint = null;
                    if (gsi.IsLeaderExist)
                    {
                        leaderHint = new Round <string>?(gsi.LeaderHint);
                    }
                    this.StateMachine = this.CreateStateMachine(leaderHint, null);
                    Task task = this.StartStateMachine();
                    task.ContinueWith(delegate(Task t)
                    {
                        this.EventLogger.Log(DxEventSeverity.Info, 0, "Successfully started state machine", new object[0]);
                        this.SnapshotManager.Start();
                        this.State = InstanceState.Running;
                        this.IsStartupCompleted = true;
                    });
                }
                else
                {
                    DxStoreInstance.Tracer.TraceWarning <string>((long)this.IdentityHash, "{0}: Instance start - waiting for the next round to start local instance", this.Identity);
                }
            }
            finally
            {
                if (flag)
                {
                    object obj;
                    Monitor.Exit(obj);
                }
            }
        }