private static void CheckIncoming(int currentTLG, SimulationContext ctx)
{
Tuple <Link, object> pair;
while (incoming.TryTake(out pair))
{
object obj = pair.Item2;
Link lnk = pair.Item1;
if (obj is ClientMessage)
{
Log.Error("Received client message via peer socket");
return;
}
if (obj is OldestGeneration)
{
int gen = ((OldestGeneration)obj).Generation;
if (gen == lnk.OldestGeneration)
{
if (siblings.AllResponsive && siblings.OldestGeneration >= gen)
{
DB.RemoveInboundRCSsAsync(neighbors.Select(sibling => sibling.InboundRCSStackID), gen).Wait(); //maybe only now responsive
}
Log.Minor("OldestGen update from sibling " + lnk + ": Warning: Already moved to generation " + gen);
return;
}
if (gen > lnk.OldestGeneration)
{
lnk.SetOldestGeneration(gen, currentTLG);
lnk.Filter((id, o) =>
{
SerialSDS sds = o as SerialSDS;
if (sds != null)
{
return(sds.Generation >= gen);
}
RCS.Serial rcs = o as RCS.Serial;
if (rcs != null)
{
return(rcs.Generation >= gen);
}
return(true);
});
if (siblings.AllResponsive && siblings.OldestGeneration >= gen)
{
DB.RemoveInboundRCSsAsync(neighbors.Select(sibling => sibling.InboundRCSStackID), gen).Wait();
}
}
return;
}
if (obj is Tuple <int, RCS> )
{
var rcs = (Tuple <int, RCS>)obj;
if (rcs.Item1 <= stack.NewestConsistentSDSGeneration)
{
Log.Error("RCS update from sibling " + lnk + ": Rejected. Already moved past generation " + rcs.Item1);
return;
}
FetchNeighborUpdate(stack.AllocateGeneration(rcs.Item1), lnk, rcs.Item2);
return;
}
if (obj is RCS.Serial)
{
RCS.Serial rcs = (RCS.Serial)obj;
if (rcs.Generation <= stack.NewestConsistentSDSGeneration)
{
Log.Error("RCS update from sibling " + lnk + ": Rejected. Already moved past generation " + rcs.Generation);
return;
}
FetchNeighborUpdate(stack.AllocateGeneration(rcs.Generation), lnk, rcs.Data);
return;
}
if (obj is SerialSDS)
{
//Debug.Assert(HaveSibling(lnk));
SerialSDS raw = (SerialSDS)obj;
if (raw.Generation <= stack.OldestSDSGeneration)
{
Log.Minor("SDS update from sibling or DB: Rejected. Already moved past generation " + raw.Generation);
return;
}
var existing = stack.FindGeneration(raw.Generation);
if (existing != null && existing.IsFullyConsistent)
{
Log.Minor("SDS update from sibling or DB: Rejected. Generation already consistent: " + raw.Generation);
return;
}
SDS sds = raw.Deserialize();
if (existing != null && existing.SDS.ICMessagesAndEntitiesAreEqual(sds))
{
Log.Minor("SDS update from sibling or DB: Equal. Ignoring");
return;
}
if (sds.IsFullyConsistent)
{
Log.Minor("SDS update from sibling or DB: Accepted generation " + raw.Generation);
stack.Insert(sds);
return;
}
SDS merged = existing.SDS.MergeWith(sds, SDS.MergeStrategy.ExclusiveWithPositionCorrection, ctx);
Log.Minor("SDS update from sibling " + lnk + ": Merged generation " + raw.Generation);
if (merged.Inconsistency < existing.SDS.Inconsistency)
{
stack.Insert(merged);
}
if (merged.Inconsistency < sds.Inconsistency)
{
lnk.Set(new SDS.ID(ID.XYZ, raw.Generation).P2PKey, new SerialSDS(merged, Simulation.ID.XYZ));
}
return;
}
if (obj is Tuple <int, MessagePack> )
{
//Debug.Assert(HaveSibling(lnk));
var m = (Tuple <int, MessagePack>)obj;
if (m.Item1 <= stack.OldestSDSGeneration)
{
Log.Minor("CCS update from sibling or DB: Rejected. Already moved past generation " + m.Item1);
return;
}
Messages.Insert(m.Item1, m.Item2);
return;
}
Log.Error("Unsupported update from sibling " + lnk + ": " + obj.GetType());
}
}
private static void RunStupidModel()
{
int gridRes = 100; //2d resolution
//each grid cell can 'see' +- 4 cells in all direction. All 'motion' is done via communication
//hence R = 4 / gridRes
float r = 4.5f / gridRes;
BaseDB.ConfigContainer config = new BaseDB.ConfigContainer()
{
extent = Int3.One, r = r, m = r * 0.5f
};
Simulation.Configure(new ShardID(Int3.Zero, 0), config, true);
Vec3 outlierCoords = Simulation.MySpace.Min;
var ctx = new SimulationContext(true);
var intermediate0 = new IntermediateSDS();
intermediate0.entities = new EntityPool(MakeGrid2D(gridRes), ctx);
//EntityTest.RandomDefaultPool(100);
intermediate0.ic = InconsistencyCoverage.NewCommon();
intermediate0.inputConsistent = true;
intermediate0.localChangeSet = new EntityChangeSet();
SDSStack.Entry root = new SDSStack.Entry(
new SDS(0, intermediate0.entities.ToArray(), intermediate0.ic),
intermediate0);
//Assert.IsTrue(root.IsFullyConsistent);
SDSStack stack = Simulation.Stack;
stack.ResetToRoot(root);
for (int i = 0; i < 13; i++)
{
//Assert.IsNotNull(stack.NewestSDS.FinalEntities, i.ToString());
var temp = stack.AllocateGeneration(i + 1);
ctx.SetGeneration(i + 1);
var comp = new SDSComputation(new DateTime(), ExtMessagePack.CompleteBlank, TimeSpan.FromMilliseconds(10), ctx);
//ComputationTests.AssertNoErrors(comp, "comp");
//Assert.IsTrue(comp.Intermediate.inputConsistent);
var sds = comp.Complete();
stack.Insert(sds);
//Assert.IsTrue(sds.IsFullyConsistent);
//Assert.AreEqual(sds.FinalEntities.Length, gridRes * gridRes);
int numBugs = 0;
int numPredators = 0;
int numConflicts = 0;
float totalFood = 0;
foreach (var e in sds.Item1.FinalEntities)
{
Habitat h = (Habitat)Helper.Deserialize(e.SerialLogicState);
if (h.bug.HasAnimal)
{
numBugs++;
}
if (h.predator.HasAnimal)
{
numPredators++;
if (h.bug.HasAnimal)
{
numConflicts++;
}
}
totalFood += h.food;
}
Console.WriteLine("Population: b=" + numBugs + ", p=" + numPredators + ", c=" + numConflicts + "; Food=" + totalFood);
}
}
public static void Run(ShardID myID)
{
//Host.Domain = ;
listener = new Listener(h => FindLink(h));
observationListener = new ObservationLink.Listener(0);
Consensus = new Consensus.Interface(
new FullShardAddress(myID, null, listener.Port, 0, observationListener.Port),
true, Interface.ThreadOperations.Everything, new DefaultNotify());
Configure(myID, BaseDB.Config, false);
AdvertiseOldestGeneration(0);
Log.Message("Polling SDS state...");
DB.Begin(myID.XYZ, s => FetchIncoming(null, s), s => FetchIncoming(null, s));
SimulationContext ctx = new SimulationContext(false);
while (true)
{
CheckIncoming(TimingInfo.Current.TopLevelGeneration, ctx);
if (stack.HasEntries)
{
break;
}
Thread.Sleep(1000);
Console.Write('.');
Console.Out.Flush();
}
var sds = stack.NewestConsistentSDS;
Messages.TrimGenerations(sds.Generation - 1);
Consensus.ForwardMessageGeneration(sds.Generation);
Log.Message(" done. Waiting for logic assemblies to finish loading...");
foreach (var e in sds.FinalEntities)
{
var logic = e.MyLogic as DynamicCSLogic;
if (logic != null)
{
logic.FinishLoading(e.ID, TimeSpan.FromMinutes(5));
}
}
Log.Message(" done");
Log.Message("Start Date=" + BaseDB.Timing.startTime);
//{
// foreach (var link in neighbors)
// DB.BeginFetch(link.InboundRCSStackID);
//}
// Log.Message("Catching up to g"+ TimingInfo.Current.TopLevelGeneration);
while (stack.NewestFinishedSDSGeneration < TimingInfo.Current.TopLevelGeneration)
{
UpdateTitle("Catching up g" + stack.NewestFinishedSDSGeneration + "/" + TimingInfo.Current.TopLevelGeneration);
//Log.Message("Catching up to g" + TimingInfo.Current.TopLevelGeneration);
Console.Write(".");
Console.Out.Flush();
int currentGen = stack.NewestFinishedSDSGeneration;
int nextGen = currentGen + 1;
ctx.SetGeneration(nextGen);
stack.Append(new SDS(nextGen));
Debug.Assert(!stack.NewestRegisteredEntry.IsFinished);
stack.Insert(new SDSComputation(Clock.Now, Messages.GetMessages(currentGen), TimingInfo.Current.EntityEvaluationTimeWindow, ctx).Complete());
Debug.Assert(stack.NewestRegisteredEntry.IsFinished);
CheckIncoming(TimingInfo.Current.TopLevelGeneration, ctx);
}
Log.Message("done. Starting main loop...");
SDSComputation mainComputation = null, recoveryComputation = null; //main computation plus one recovery computation max
int lastRecoveryIndex = -1;
while (true)
{
var timing = TimingInfo.Current;
CheckIncoming(timing.TopLevelGeneration, ctx);
Log.Minor("TLG " + stack.NewestFinishedSDSGeneration + "/" + timing.TopLevelGeneration + " @recoveryStepIndex " + timing.LatestRecoveryStepIndex);
{
var newest = stack.NewestFinishedSDS;
string title = ID + " g" + newest.Generation + " " + (float)(newest.IC.Size.Product - newest.IC.OneCount) * 100 / newest.IC.Size.Product + "% consistent";
var con = stack.NewestConsistentSDS;
if (con != newest)
{
title += ", newest consistent at g" + con.Generation;
}
title += ", rec " + timing.LatestRecoveryStepIndex;
UpdateTitle(title);
}
int newestSDSGeneration = stack.NewestFinishedSDSGeneration;
if (mainComputation == null)
{
Debug.Assert(stack.NewestRegisteredEntry.IsFinished);
Debug.Assert(newestSDSGeneration == stack.NewestRegisteredSDSGeneration);
Debug.Assert(stack.NewestConsistentSDSIndex != -1);
}
if (recoveryComputation != null && Complete(recoveryComputation, timing, timing.TopLevelGeneration != newestSDSGeneration))
{
recoveryComputation = null;
}
if (mainComputation != null && Complete(mainComputation, timing, timing.TopLevelGeneration != newestSDSGeneration && timing.TopLevelGeneration > mainComputation.Generation))
{
newestSDSGeneration = stack.NewestFinishedSDSGeneration;
Debug.Assert(stack.NewestRegisteredEntry.IsFinished);
Debug.Assert(newestSDSGeneration == stack.NewestRegisteredSDSGeneration);
Debug.Assert(stack.NewestConsistentSDSIndex != -1);
mainComputation = null;
}
if (recoveryComputation != null && mainComputation != null)
{
Clock.SleepUntil(Helper.Min(recoveryComputation.Deadline, mainComputation.Deadline));
}
else
if (recoveryComputation != null)
{
Clock.SleepUntil(recoveryComputation.Deadline);
}
else
if (mainComputation != null)
{
Clock.SleepUntil(mainComputation.Deadline);
}
if (recoveryComputation != null) //recovery computations are analogue to main computation, so main computation will not be done. but recovery must be
{
continue;
}
if (mainComputation == null && timing.TopLevelGeneration > newestSDSGeneration)
{
//fast forward: process now. don't care if we're at the beginning
Debug.Assert(stack.NewestRegisteredEntry.IsFinished);
Debug.Assert(newestSDSGeneration == stack.NewestRegisteredSDSGeneration);
Debug.Assert(stack.NewestConsistentSDSIndex != -1);
int nextGen = newestSDSGeneration + 1;
Log.Message("Processing next TLG g" + nextGen);
stack.Insert(new SDS(nextGen));
ctx.SetGeneration(nextGen);
Debug.Assert(mainComputation == null);
Consensus.ForceCommitGECIfLeader(newestSDSGeneration);
mainComputation = new SDSComputation(timing.NextMainApplicationDeadline, Messages.GetMessages(newestSDSGeneration), timing.EntityEvaluationTimeWindow, ctx);
}
if (timing.ShouldStartRecovery(ref lastRecoveryIndex))
{
//see if we can recover something
int oldestInconsistentSDSIndex = stack.NewestConsistentSDSIndex + 1; //must be > 0
int top = Math.Min(stack.Size - 1, stack.ToIndex(timing.TopLevelGeneration));
if (oldestInconsistentSDSIndex <= top)
{
int end = mainComputation == null ? top + 1 : top;
int recoverAtIndex = oldestInconsistentSDSIndex;
int currentGen = stack[recoverAtIndex].Generation;
for (; recoverAtIndex < end; recoverAtIndex++)
{
var current = stack[recoverAtIndex];
if (current.SignificantInboundChange)
{
break;
}
var check = CheckMissingRCS(current);
if (check.ShouldRecoverThis)
{
break;
}
}
if (recoverAtIndex < end)
{
var deadline = timing.GetRecoveryStepApplicationDeadline(lastRecoveryIndex);
Log.Message("Recovering #" + recoverAtIndex + "/" + top + ", g" + stack[recoverAtIndex].Generation + ", deadline=" + deadline);
//precompute:
ctx.SetGeneration(stack[recoverAtIndex].Generation);
recoveryComputation = new SDSComputation(deadline, Messages.GetMessages(ctx.GenerationNumber - 1), timing.EntityEvaluationTimeWindow, ctx);
//now wait for remote RCS...
}
}
}
}
}
