internal void InstallConsensusCluster(int size, int basePort, bool awaitFormation) { BaseDB.OverrideAddressRequestFunction = addr => new FullShardAddress(addr, "localhost", 0, basePort + addr.ReplicaLevel, basePort + addr.ReplicaLevel + 100); BaseDB.SDConfigPoller = new ReplicaOnlySD(size); var dummyNotify = new DummyNotify(); consensus = new Consensus.Interface[size]; Consensus.SharedDebugState state = new Consensus.SharedDebugState(); for (int i = 0; i < size; i++) { consensus[i] = new Consensus.Interface( new Consensus.Configuration.Member(i, true), new Address(basePort + i), Int3.Zero, Interface.ThreadOperations.Everything, i > 0 ? dummyNotify : Notify) { DebugState = state }; } if (awaitFormation) { AwaitConsensus(); } }
public static void Run(ShardID myID) { //Host.Domain = ; listener = new Listener(h => FindLink(h)); observationListener = new ObservationLink.Listener(0); Consensus = new Consensus.Interface( new FullShardAddress(myID, null, listener.Port, 0, observationListener.Port), true, Interface.ThreadOperations.Everything, new DefaultNotify()); Configure(myID, BaseDB.Config, false); AdvertiseOldestGeneration(0); Log.Message("Polling SDS state..."); DB.Begin(myID.XYZ, s => FetchIncoming(null, s), s => FetchIncoming(null, s)); SimulationContext ctx = new SimulationContext(false); while (true) { CheckIncoming(TimingInfo.Current.TopLevelGeneration, ctx); if (stack.HasEntries) { break; } Thread.Sleep(1000); Console.Write('.'); Console.Out.Flush(); } var sds = stack.NewestConsistentSDS; Messages.TrimGenerations(sds.Generation - 1); Consensus.ForwardMessageGeneration(sds.Generation); Log.Message(" done. Waiting for logic assemblies to finish loading..."); foreach (var e in sds.FinalEntities) { var logic = e.MyLogic as DynamicCSLogic; if (logic != null) { logic.FinishLoading(e.ID, TimeSpan.FromMinutes(5)); } } Log.Message(" done"); Log.Message("Start Date=" + BaseDB.Timing.startTime); //{ // foreach (var link in neighbors) // DB.BeginFetch(link.InboundRCSStackID); //} // Log.Message("Catching up to g"+ TimingInfo.Current.TopLevelGeneration); while (stack.NewestFinishedSDSGeneration < TimingInfo.Current.TopLevelGeneration) { UpdateTitle("Catching up g" + stack.NewestFinishedSDSGeneration + "/" + TimingInfo.Current.TopLevelGeneration); //Log.Message("Catching up to g" + TimingInfo.Current.TopLevelGeneration); Console.Write("."); Console.Out.Flush(); int currentGen = stack.NewestFinishedSDSGeneration; int nextGen = currentGen + 1; ctx.SetGeneration(nextGen); stack.Append(new SDS(nextGen)); Debug.Assert(!stack.NewestRegisteredEntry.IsFinished); stack.Insert(new SDSComputation(Clock.Now, Messages.GetMessages(currentGen), TimingInfo.Current.EntityEvaluationTimeWindow, ctx).Complete()); Debug.Assert(stack.NewestRegisteredEntry.IsFinished); CheckIncoming(TimingInfo.Current.TopLevelGeneration, ctx); } Log.Message("done. Starting main loop..."); SDSComputation mainComputation = null, recoveryComputation = null; //main computation plus one recovery computation max int lastRecoveryIndex = -1; while (true) { var timing = TimingInfo.Current; CheckIncoming(timing.TopLevelGeneration, ctx); Log.Minor("TLG " + stack.NewestFinishedSDSGeneration + "/" + timing.TopLevelGeneration + " @recoveryStepIndex " + timing.LatestRecoveryStepIndex); { var newest = stack.NewestFinishedSDS; string title = ID + " g" + newest.Generation + " " + (float)(newest.IC.Size.Product - newest.IC.OneCount) * 100 / newest.IC.Size.Product + "% consistent"; var con = stack.NewestConsistentSDS; if (con != newest) { title += ", newest consistent at g" + con.Generation; } title += ", rec " + timing.LatestRecoveryStepIndex; UpdateTitle(title); } int newestSDSGeneration = stack.NewestFinishedSDSGeneration; if (mainComputation == null) { Debug.Assert(stack.NewestRegisteredEntry.IsFinished); Debug.Assert(newestSDSGeneration == stack.NewestRegisteredSDSGeneration); Debug.Assert(stack.NewestConsistentSDSIndex != -1); } if (recoveryComputation != null && Complete(recoveryComputation, timing, timing.TopLevelGeneration != newestSDSGeneration)) { recoveryComputation = null; } if (mainComputation != null && Complete(mainComputation, timing, timing.TopLevelGeneration != newestSDSGeneration && timing.TopLevelGeneration > mainComputation.Generation)) { newestSDSGeneration = stack.NewestFinishedSDSGeneration; Debug.Assert(stack.NewestRegisteredEntry.IsFinished); Debug.Assert(newestSDSGeneration == stack.NewestRegisteredSDSGeneration); Debug.Assert(stack.NewestConsistentSDSIndex != -1); mainComputation = null; } if (recoveryComputation != null && mainComputation != null) { Clock.SleepUntil(Helper.Min(recoveryComputation.Deadline, mainComputation.Deadline)); } else if (recoveryComputation != null) { Clock.SleepUntil(recoveryComputation.Deadline); } else if (mainComputation != null) { Clock.SleepUntil(mainComputation.Deadline); } if (recoveryComputation != null) //recovery computations are analogue to main computation, so main computation will not be done. but recovery must be { continue; } if (mainComputation == null && timing.TopLevelGeneration > newestSDSGeneration) { //fast forward: process now. don't care if we're at the beginning Debug.Assert(stack.NewestRegisteredEntry.IsFinished); Debug.Assert(newestSDSGeneration == stack.NewestRegisteredSDSGeneration); Debug.Assert(stack.NewestConsistentSDSIndex != -1); int nextGen = newestSDSGeneration + 1; Log.Message("Processing next TLG g" + nextGen); stack.Insert(new SDS(nextGen)); ctx.SetGeneration(nextGen); Debug.Assert(mainComputation == null); Consensus.ForceCommitGECIfLeader(newestSDSGeneration); mainComputation = new SDSComputation(timing.NextMainApplicationDeadline, Messages.GetMessages(newestSDSGeneration), timing.EntityEvaluationTimeWindow, ctx); } if (timing.ShouldStartRecovery(ref lastRecoveryIndex)) { //see if we can recover something int oldestInconsistentSDSIndex = stack.NewestConsistentSDSIndex + 1; //must be > 0 int top = Math.Min(stack.Size - 1, stack.ToIndex(timing.TopLevelGeneration)); if (oldestInconsistentSDSIndex <= top) { int end = mainComputation == null ? top + 1 : top; int recoverAtIndex = oldestInconsistentSDSIndex; int currentGen = stack[recoverAtIndex].Generation; for (; recoverAtIndex < end; recoverAtIndex++) { var current = stack[recoverAtIndex]; if (current.SignificantInboundChange) { break; } var check = CheckMissingRCS(current); if (check.ShouldRecoverThis) { break; } } if (recoverAtIndex < end) { var deadline = timing.GetRecoveryStepApplicationDeadline(lastRecoveryIndex); Log.Message("Recovering #" + recoverAtIndex + "/" + top + ", g" + stack[recoverAtIndex].Generation + ", deadline=" + deadline); //precompute: ctx.SetGeneration(stack[recoverAtIndex].Generation); recoveryComputation = new SDSComputation(deadline, Messages.GetMessages(ctx.GenerationNumber - 1), timing.EntityEvaluationTimeWindow, ctx); //now wait for remote RCS... } } } } }