public SerialSDS(SDS sds, Int3 sectorID)
{
SerialEntities = Entity.Export(sds.FinalEntities);
Generation = sds.Generation;
IC = sds.IC.Export();
_id = sectorID.Encoded;
}
public void SendSDS(SDS sds)
{
if (closed)
{
return;
}
SendNewProvidersOf(sds);
SendCompressed(sds);
}
public static void SignalUpdate(SDS sds)
{
lock (registry)
{
foreach (var o in registry)
{
o.SendSDS(sds);
}
}
}
public SDS MergeWith(SDS other, MergeStrategy strategy, EntityChange.ExecutionContext ctx)
{
if (Generation != other.Generation)
{
throw new IntegrityViolation("Generation mismatch: " + Generation + " != " + other.Generation);
}
SDS exclusiveSource = null;
int exclusiveChoice = 0;
if (strategy == MergeStrategy.Exclusive || strategy == MergeStrategy.ExclusiveWithPositionCorrection)
{
exclusiveChoice = SelectExclusiveSource(this, other);
exclusiveSource = (exclusiveChoice == -1 ? this : other);
}
InconsistencyCoverage merged = InconsistencyCoverage.GetMinimum(IC, other.IC);
EntityPool pool = new EntityPool(ctx);
foreach (var e in this.FinalEntities)
{
if (IC.IsInconsistentR(ctx.LocalSpace.Relativate(e.ID.Position)))
{
continue; //for now
}
pool.Insert(e);
}
foreach (var e in other.FinalEntities)
{
if (other.IC.IsInconsistentR(ctx.LocalSpace.Relativate(e.ID.Position)))
{
continue; //for now
}
if (pool.Contains(e.ID.Guid))
{
continue;
}
pool.Insert(e);
}
//at this point we merged all fully consistent entities from either. If the inconsistent areas did not overlap then the result should contain all entities in their consistent state
if (!merged.IsFullyConsistent)
{
if (strategy == MergeStrategy.EntitySelective)
{
MergeInconsistentEntitiesComp(pool, this, other, merged, ctx);
}
else
{
MergeInconsistentEntitiesEx(pool, exclusiveSource, strategy == MergeStrategy.ExclusiveWithPositionCorrection, merged, ctx);
}
}
return(new SDS(Generation, pool.ToArray(), merged));
}
private static int SelectExclusiveSource(SDS a, SDS b)
{
int balance = 0;
var e0 = a.IC.Bits.GetEnumerator();
var e1 = b.IC.Bits.GetEnumerator();
while (e0.MoveNext() && e1.MoveNext())
{
balance += e0.Current.CompareTo(e1.Current);
}
return(balance < 0 ? -1 : 1);
}
private void SendNewProvidersOf(SDS sds)
{
//Message("Checking providers of g" + sds.Generation);
foreach (var e in sds.FinalEntities)
{
DynamicCSLogic logic = e.MyLogic as DynamicCSLogic;
if (logic == null)
{
continue;
}
if (string.IsNullOrEmpty(logic.Provider.AssemblyName))
{
throw new IntegrityViolation("");
}
//Message("Checking logic " + logic.Provider);
SendProvider(logic.Provider);
}
}
public Tuple <SDS, IntermediateSDS> Complete()
{
//Log.Message("Finalize SDS g" + generation);
var cs = data.localChangeSet.Clone();
InconsistencyCoverage ic = data.ic.Clone();
foreach (var n in Simulation.Neighbors)
{
IntBox box = n.ICImportRegion;
var rcs = old.InboundRCS[n.LinearIndex];
if (rcs != null)
{
cs.Include(rcs.CS);
ic.Include(rcs.IC, box.Min);
if (rcs.IC.OneCount > 0)
{
Log.Message(n.Name + ": Inconsistent RCS @g" + generation + ": " + rcs.IC.OneCount);
}
}
else
{
Log.Message(n.Name + ": Missing RCS @g" + generation);
ic.SetOne(box);
}
}
EntityPool p2 = data.entities.Clone();
cs.Execute(p2, ic, ctx);
SDS rs = new SDS(generation, p2.ToArray(), ic);
#if !DRY_RUN
if (!ic.AnySet)
{
DB.PutAsync(new SerialSDS(rs, Simulation.ID.XYZ), false).Wait();
}
#endif
Log.Message("Completed g" + Generation + " with IC ones: " + ic.OneCount + " " + Math.Round((float)ic.OneCount / ic.Size.Product * 100) + "%");
return(new Tuple <SDS, IntermediateSDS>(rs, data));
}
public static byte[] Compress(SDS sds)
{
var serial = new Newtonsoft.Json.JsonSerializer();
StringBuilder str = new StringBuilder();
TextWriter w = new StringWriter(str);
foreach (var e in sds.FinalEntities)
{
serial.Serialize(w, e);
}
using (MemoryStream ms = new MemoryStream())
using (var stream = new LZ4.LZ4Stream(Helper.Serialize(sds), LZ4.LZ4StreamMode.Compress))
{
stream.CopyTo(ms);
return(ms.ToArray());
}
}
private static void MergeInconsistentEntitiesEx(EntityPool pool, SDS source, bool correctLocations, InconsistencyCoverage ic, EntityChange.ExecutionContext ctx)
{
const float searchScope = 0.5f;
foreach (var e0 in source.FinalEntities)
{
if (pool.Contains(e0.ID.Guid))
{
continue; //already good
}
var c0 = ctx.LocalSpace.Relativate(e0.ID.Position);
if (ic.IsInconsistentR(c0))
{
pool.Insert(e0);
}
else
{
if (correctLocations)
{
var c = c0;
if (ic.FindInconsistentPlacementCandidateR(ref c, searchScope))
{
Entity me = e0.Relocate(c);
pool.Insert(me);
}
}
//only increases overaccounted entities:
//if (merged.ic.FindInconsistentPlacementCandidate(c0,searchScope))
//{
// Entity me = *e0;
// me.coordinates = c0 + shardOffset;
// merged.entities.InsertEntity(me);
//}
}
}
}
public Entry(SDS sds, IntermediateSDS intermediate) : this(sds)
{
IntermediateSDS = intermediate;
}
public Entry(SDS sds)
{
SDS = sds;
Generation = sds.Generation;
}
public void Append(SDS sds)
{
Append(new Entry(sds));
}
public void Insert(SDS sds, bool trim = true)
{
Insert(new Entry(sds), trim);
}
public bool ICMessagesAndEntitiesAreEqual(SDS other)
{
return(IC.Equals(other.IC) &&
Helper.AreEqual(FinalEntities, other.FinalEntities)
);
}
private static void MergeInconsistentEntitiesComp(EntityPool pool, SDS s0, SDS s1, InconsistencyCoverage ic, EntityChange.ExecutionContext ctx)
{
var a = new EntityPool(s0.FinalEntities, ctx);
var b = new EntityPool(s1.FinalEntities, ctx);
const float searchScope = 0.5f;
foreach (var e0 in a)
{
if (pool.Contains(e0.ID.Guid))
{
continue; //already good
}
//entity is inconsistent and not in merged state yet
var c0 = ctx.LocalSpace.Relativate(e0.ID.Position);
var e1 = b.Find(e0.ID.Guid);
var c1 = e1 != null?ctx.LocalSpace.Relativate(e1.ID.Position) : Vec3.Zero;
if (!ic.IsInconsistentR(c0))
{
//this is tricky. entity not merged, but would reside in consistent space (bad).
if (e1 != null)
{
// ASSERT__(b.ic.IsInconsistent(c1)); //otherwise it would have been added prior, and we would not be here
//so this entity exists in both SDS'
{
//we now have the same entity twice, both inconsistent, residing each in the consistent space of the other SDS'
//let's assume the entity isn't supposed to exist here anyways
Entity candidate = null;
int sc = s0.IC.GetInconsistencyAtR(c0).CompareTo(s1.IC.GetInconsistencyAtR(c1));
if (sc < 0)
{
candidate = e0;
}
else if (sc > 0)
{
candidate = e1;
}
else if (e0.CompareTo(e1) < 0)
{
candidate = e0;
}
else
{
candidate = e1;
}
var c = ctx.LocalSpace.Relativate(candidate.ID.Position);
if (ic.FindInconsistentPlacementCandidateR(ref c, searchScope))
{
Entity me = candidate.Relocate(ctx.LocalSpace.DeRelativate(c));
pool.Insert(me);
}
}
}
else
{
//entity exists only in local SDS.
//let's assume the entity isn't supposed to exist here anyways
//TEntityCoords c = Frac(e0->coordinates);
//if (merged.ic.FindInconsistentPlacementCandidate(c,searchScope))
//{
// Entity copy = *e0;
// copy.coordinates = c + shardOffset;
// //ASSERT__(merged.ic.IsInconsistent(Frac(copy.coordinates)));
// merged.entities.InsertEntity(copy);
//}
//else
// FATAL__("bad");
}
}
else
{
//entity location is inconsistent in both SDS'. This is expected to be the most common case
if (e1 != null)
{
if (e1.Equals(e0))
{
//probably actually consistent
// ASSERT__(merged.ic.IsInconsistent(Frac(e0->coordinates)));
pool.Insert(e0);
}
else
{
if (!ic.IsInconsistentR(c1))
{
Debug.Assert(ic.IsInconsistentR(c0));
pool.Insert(e0);
}
else
{
Entity candidate = null;
int sc = s0.IC.GetInconsistencyAtR(c0).CompareTo(s1.IC.GetInconsistencyAtR(c1));
if (sc < 0)
{
candidate = e0;
}
else if (sc > 0)
{
candidate = e1;
}
else if (e0.CompareTo(e1) < 0)
{
candidate = e0;
}
else
{
candidate = e1;
}
//common case. Choose one
//ASSERT__(ic.IsInconsistentR(candidate->coordinates-shardOffset));
pool.Insert(candidate);
}
}
}
else
{
//only e0 exists
int sc = s0.IC.GetInconsistencyAtR(c0).CompareTo(s1.IC.GetInconsistencyAtR(c0));
//ASSERT__(merged.ic.IsInconsistent(Frac(e0->coordinates)));
if (sc <= 0)
{
pool.Insert(e0);
}
}
}
}
foreach (var e0 in b)
{
if (pool.Contains(e0.ID.Guid))
{
continue; //already good
}
//entity is inconsistent and not in merged state yet
var c0 = ctx.LocalSpace.Relativate(e0.ID.Position);
//const auto c1 = e1 ? (e1->coordinates - shardOffset) : TEntityCoords();
if (!ic.IsInconsistentR(c0))
{
#if false
//this is tricky. entity not merged, but would reside in consistent space (bad).
if (e1)
{
//case handled
//FATAL__("bad");
}
else
{
//entity exists only in local SDS.
//let's assume the entity isn't supposed to exist here anyways
//TEntityCoords c = Frac(e0->coordinates);
//if (merged.ic.FindInconsistentPlacementCandidate(c,searchScope))
//{
// Entity copy = *e0;
// copy.coordinates = c + shardOffset;
// //ASSERT__(merged.ic.IsInconsistent(Frac(copy.coordinates)));
// merged.entities.InsertEntity(copy);
//}
/* else
* FATAL__("bad");*/
}
#endif
}
else
{
var e1 = a.Find(e0.ID.Guid);
//entity location is inconsistent in both SDS'. This is expected to be the most common case
if (e1 != null)
{
//case handled
//FATAL__("bad");
}
else
{
//only e0 exists
int sc = s0.IC.GetInconsistencyAtR(c0).CompareTo(s1.IC.GetInconsistencyAtR(c0));
//ASSERT__(merged.ic.IsInconsistent(Frac(e0->coordinates)));
if (sc >= 0)
{
pool.Insert(e0);
}
}
}
}
}
private static void CheckIncoming(int currentTLG, SimulationContext ctx)
{
Tuple <Link, object> pair;
while (incoming.TryTake(out pair))
{
object obj = pair.Item2;
Link lnk = pair.Item1;
if (obj is ClientMessage)
{
Log.Error("Received client message via peer socket");
return;
}
if (obj is OldestGeneration)
{
int gen = ((OldestGeneration)obj).Generation;
if (gen == lnk.OldestGeneration)
{
if (siblings.AllResponsive && siblings.OldestGeneration >= gen)
{
DB.RemoveInboundRCSsAsync(neighbors.Select(sibling => sibling.InboundRCSStackID), gen).Wait(); //maybe only now responsive
}
Log.Minor("OldestGen update from sibling " + lnk + ": Warning: Already moved to generation " + gen);
return;
}
if (gen > lnk.OldestGeneration)
{
lnk.SetOldestGeneration(gen, currentTLG);
lnk.Filter((id, o) =>
{
SerialSDS sds = o as SerialSDS;
if (sds != null)
{
return(sds.Generation >= gen);
}
RCS.Serial rcs = o as RCS.Serial;
if (rcs != null)
{
return(rcs.Generation >= gen);
}
return(true);
});
if (siblings.AllResponsive && siblings.OldestGeneration >= gen)
{
DB.RemoveInboundRCSsAsync(neighbors.Select(sibling => sibling.InboundRCSStackID), gen).Wait();
}
}
return;
}
if (obj is Tuple <int, RCS> )
{
var rcs = (Tuple <int, RCS>)obj;
if (rcs.Item1 <= stack.NewestConsistentSDSGeneration)
{
Log.Error("RCS update from sibling " + lnk + ": Rejected. Already moved past generation " + rcs.Item1);
return;
}
FetchNeighborUpdate(stack.AllocateGeneration(rcs.Item1), lnk, rcs.Item2);
return;
}
if (obj is RCS.Serial)
{
RCS.Serial rcs = (RCS.Serial)obj;
if (rcs.Generation <= stack.NewestConsistentSDSGeneration)
{
Log.Error("RCS update from sibling " + lnk + ": Rejected. Already moved past generation " + rcs.Generation);
return;
}
FetchNeighborUpdate(stack.AllocateGeneration(rcs.Generation), lnk, rcs.Data);
return;
}
if (obj is SerialSDS)
{
//Debug.Assert(HaveSibling(lnk));
SerialSDS raw = (SerialSDS)obj;
if (raw.Generation <= stack.OldestSDSGeneration)
{
Log.Minor("SDS update from sibling or DB: Rejected. Already moved past generation " + raw.Generation);
return;
}
var existing = stack.FindGeneration(raw.Generation);
if (existing != null && existing.IsFullyConsistent)
{
Log.Minor("SDS update from sibling or DB: Rejected. Generation already consistent: " + raw.Generation);
return;
}
SDS sds = raw.Deserialize();
if (existing != null && existing.SDS.ICMessagesAndEntitiesAreEqual(sds))
{
Log.Minor("SDS update from sibling or DB: Equal. Ignoring");
return;
}
if (sds.IsFullyConsistent)
{
Log.Minor("SDS update from sibling or DB: Accepted generation " + raw.Generation);
stack.Insert(sds);
return;
}
SDS merged = existing.SDS.MergeWith(sds, SDS.MergeStrategy.ExclusiveWithPositionCorrection, ctx);
Log.Minor("SDS update from sibling " + lnk + ": Merged generation " + raw.Generation);
if (merged.Inconsistency < existing.SDS.Inconsistency)
{
stack.Insert(merged);
}
if (merged.Inconsistency < sds.Inconsistency)
{
lnk.Set(new SDS.ID(ID.XYZ, raw.Generation).P2PKey, new SerialSDS(merged, Simulation.ID.XYZ));
}
return;
}
if (obj is Tuple <int, MessagePack> )
{
//Debug.Assert(HaveSibling(lnk));
var m = (Tuple <int, MessagePack>)obj;
if (m.Item1 <= stack.OldestSDSGeneration)
{
Log.Minor("CCS update from sibling or DB: Rejected. Already moved past generation " + m.Item1);
return;
}
Messages.Insert(m.Item1, m.Item2);
return;
}
Log.Error("Unsupported update from sibling " + lnk + ": " + obj.GetType());
}
}
