/// <summary>
/// Creates an event object based on a given type specifier.
/// </summary>
/// <param name="id">Type specifier.</param>
/// <returns>A specialized event object.</returns>
public EventBase CreateEvent(int id)
{
EventBase result;
Type eventType = m_events.Find(id);
System.Diagnostics.Debug.Assert(eventType != null, "Event type not found id: " + id);
System.Diagnostics.Debug.Assert(eventType.IsSubclassOf(typeof(EventBase)), "Event doesn't inherit from EventBase id: " + id);
result = (EventBase)Activator.CreateInstance(eventType);
return(result);
}
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch)
{
var count = batch.Count;
var srckey = batch.key.col;
fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col)
{
fixed(int *src_hash = batch.hash.col)
{
for (int i = 0; i < count; i++)
{
if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
{
var partitionKey = this.getPartitionKey(srckey[i]);
int partitionIndex = EnsurePartition(partitionKey);
long synctime = src_vsync[i];
int index;
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
ProcessCurrentTimestamp(partitionIndex);
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
bool done = false;
var eventListTraverser = new FastMap <SavedEventList <TKey, TPayload> > .FindTraverser(this.currentTimestampEventList.entries[partitionIndex].value);
if (eventListTraverser.Find(src_hash[i]))
{
while (eventListTraverser.Next(out index))
{
var state = this.currentTimestampEventList.entries[partitionIndex].value.Values[index];
if (this.keyEqualityComparer(state.key, srckey[i]))
{
state.payloads.Add(batch.payload.col[i]);
done = true;
break;
}
}
}
if (!done)
{
index = this.currentTimestampEventList.entries[partitionIndex].value.Insert(src_hash[i]);
var list = new List <TPayload>(10)
{
batch[i]
};
this.currentTimestampEventList.entries[partitionIndex].value.Values[index] = new SavedEventList <TKey, TPayload> {
key = srckey[i], payloads = list
};
}
}
else if (src_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime)
{
long synctime = src_vsync[i];
int partitionIndex = FastDictionary2 <TPartitionKey, List <TKey> > .IteratorStart;
while (this.lastSyncTime.Iterate(ref partitionIndex))
{
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
ProcessCurrentTimestamp(partitionIndex);
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
}
OnLowWatermark(synctime);
}
else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime)
{
var partitionKey = this.getPartitionKey(srckey[i]);
int partitionIndex = EnsurePartition(partitionKey);
long synctime = src_vsync[i];
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
ProcessCurrentTimestamp(partitionIndex);
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
}
}
}
}
batch.Free();
}
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch)
{
var stack = new Stack <int>();
var count = batch.Count;
var dest_vsync = this.batch.vsync.col;
var dest_vother = this.batch.vother.col;
var destkey = this.batch.key.col;
var dest_hash = this.batch.hash.col;
var srckey = batch.key.col;
var activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates);
fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col)
{
fixed(int *src_hash = batch.hash.col)
{
for (int i = 0; i < count; i++)
{
if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
{
var key = srckey[i];
var partitionKey = this.getPartitionKey(key);
int partitionIndex = EnsurePartition(partitionKey);
var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput.entries[partitionIndex].value);
long synctime = src_vsync[i];
if (!this.IsSyncTimeSimultaneityFree)
{
int index;
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
foreach (var mapIndex in this.seenPartitions.entries[partitionIndex].value)
{
this.seenEvent.Remove(mapIndex);
}
if (this.tentativeOutput.Count > 0)
{
tentativeVisibleTraverser.currIndex = 0;
while (tentativeVisibleTraverser.Next(out index, out int hash))
{
var elem = this.tentativeOutput.entries[partitionIndex].value.Values[index];
dest_vsync[this.iter] = this.lastSyncTime.entries[partitionIndex].value;
dest_vother[this.iter] = elem.other;
this.batch.payload.col[this.iter] = elem.payload;
destkey[this.iter] = elem.key;
dest_hash[this.iter] = hash;
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
this.tentativeOutput.entries[partitionIndex].value.Clear(); // Clear the tentative output list
}
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
if (this.seenEvent.Lookup(srckey[i], out index)) // Incoming event is a simultaneous one
{
if (this.seenEvent.entries[index].value == 1) // Detecting first duplicate, need to adjust state
{
this.seenEvent.entries[index].value = 2;
// Delete tentative output for that key
var tentativeFindTraverser = new FastMap <OutputEvent <TKey, TRegister> > .FindTraverser(this.tentativeOutput.entries[partitionIndex].value);
if (tentativeFindTraverser.Find(src_hash[i]))
{
while (tentativeFindTraverser.Next(out index))
{
if (this.keyEqualityComparer(this.tentativeOutput.entries[partitionIndex].value.Values[index].key, srckey[i]))
{
tentativeFindTraverser.Remove();
}
}
}
// Delete active states for that key
if (activeFindTraverser.Find(src_hash[i]))
{
while (activeFindTraverser.Next(out index))
{
if (this.keyEqualityComparer(this.activeStates.Values[index].key, srckey[i]))
{
activeFindTraverser.Remove();
}
}
}
}
// Dont process this event
continue;
}
else
{
this.seenEvent.Insert(srckey[i], 1);
}
}
/* (1) Process currently active states */
bool ended = true;
if (activeFindTraverser.Find(src_hash[i]))
{
int orig_index;
while (activeFindTraverser.Next(out int index))
{
orig_index = index;
var state = this.activeStates.Values[index];
if (!this.keyEqualityComparer(state.key, srckey[i]))
{
continue;
}
if (state.PatternStartTimestamp + this.MaxDuration > synctime)
{
var currentStateMap = this.singleEventStateMap[state.state];
if (currentStateMap != null)
{
var m = currentStateMap.Length;
for (int cnt = 0; cnt < m; cnt++)
{
var arcinfo = currentStateMap[cnt];
if (arcinfo.Fence(synctime, batch[i], state.register))
{
TRegister newReg;
if (arcinfo.Transfer == null)
{
newReg = state.register;
}
else
{
newReg = arcinfo.Transfer(synctime, batch[i], state.register);
}
int ns = arcinfo.toState;
while (true)
{
if (this.isFinal[ns])
{
if (!this.IsSyncTimeSimultaneityFree)
{
var tentativeOutputEntry = this.tentativeOutput.entries[partitionIndex].value;
int ind = tentativeOutputEntry.Insert(src_hash[i]);
tentativeOutputEntry.Values[ind].other = state.PatternStartTimestamp + this.MaxDuration;
tentativeOutputEntry.Values[ind].key = srckey[i];
tentativeOutputEntry.Values[ind].payload = newReg;
}
else
{
dest_vsync[this.iter] = synctime;
dest_vother[this.iter] = state.PatternStartTimestamp + this.MaxDuration;
this.batch[this.iter] = newReg;
destkey[this.iter] = srckey[i];
dest_hash[this.iter] = src_hash[i];
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
}
if (this.hasOutgoingArcs[ns])
{
if (index == -1)
{
index = this.activeStates.Insert(src_hash[i]);
}
this.activeStates.Values[index].key = srckey[i];
this.activeStates.Values[index].state = ns;
this.activeStates.Values[index].register = newReg;
this.activeStates.Values[index].PatternStartTimestamp = state.PatternStartTimestamp;
index = -1;
ended = false;
// Add epsilon arc destinations to stack
if (this.epsilonStateMap == null)
{
break;
}
if (this.epsilonStateMap[ns] != null)
{
for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++)
{
stack.Push(this.epsilonStateMap[ns][cnt2]);
}
}
}
if (stack.Count == 0)
{
break;
}
ns = stack.Pop();
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one successful transition
}
}
}
}
}
if (index == orig_index)
{
activeFindTraverser.Remove();
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one active state
}
}
}
/* (2) Start new activations from the start state(s) */
if (!this.AllowOverlappingInstances && !ended)
{
continue;
}
for (int counter = 0; counter < this.numStartStates; counter++)
{
int startState = this.startStates[counter];
var startStateMap = this.singleEventStateMap[startState];
if (startStateMap != null)
{
var m = startStateMap.Length;
for (int cnt = 0; cnt < m; cnt++)
{
var arcinfo = startStateMap[cnt];
if (arcinfo.Fence(synctime, batch[i], this.defaultRegister))
{
TRegister newReg;
if (arcinfo.Transfer == null)
{
newReg = this.defaultRegister;
}
else
{
newReg = arcinfo.Transfer(synctime, batch[i], this.defaultRegister);
}
int ns = arcinfo.toState;
while (true)
{
if (this.isFinal[ns])
{
if (!this.IsSyncTimeSimultaneityFree)
{
var tentativeOutputEntry = this.tentativeOutput.entries[partitionIndex].value;
int ind = tentativeOutputEntry.Insert(src_hash[i]);
tentativeOutputEntry.Values[ind].other = synctime + this.MaxDuration;
tentativeOutputEntry.Values[ind].key = srckey[i];
tentativeOutputEntry.Values[ind].payload = newReg;
}
else
{
dest_vsync[this.iter] = synctime;
dest_vother[this.iter] = synctime + this.MaxDuration;
this.batch[this.iter] = newReg;
destkey[this.iter] = srckey[i];
dest_hash[this.iter] = src_hash[i];
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
}
if (this.hasOutgoingArcs[ns])
{
int index = this.activeStates.Insert(src_hash[i]);
this.activeStates.Values[index].key = srckey[i];
this.activeStates.Values[index].state = ns;
this.activeStates.Values[index].register = newReg;
this.activeStates.Values[index].PatternStartTimestamp = synctime;
// Add epsilon arc destinations to stack
if (this.epsilonStateMap == null)
{
break;
}
if (this.epsilonStateMap[ns] != null)
{
for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++)
{
stack.Push(this.epsilonStateMap[ns][cnt2]);
}
}
}
if (stack.Count == 0)
{
break;
}
ns = stack.Pop();
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one successful transition
}
}
}
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one start state
}
}
}
else if (src_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime)
{
long synctime = src_vsync[i];
if (!this.IsSyncTimeSimultaneityFree)
{
var partitionIndex = FastDictionary2 <TPartitionKey, long> .IteratorStart;
while (this.tentativeOutput.Iterate(ref partitionIndex))
{
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput.entries[partitionIndex].value);
foreach (var mapIndex in this.seenPartitions.entries[partitionIndex].value)
{
this.seenEvent.Remove(mapIndex);
}
if (this.tentativeOutput.Count > 0)
{
tentativeVisibleTraverser.currIndex = 0;
while (tentativeVisibleTraverser.Next(out int index, out int hash))
{
var elem = this.tentativeOutput.entries[partitionIndex].value.Values[index];
this.batch.vsync.col[this.iter] = this.lastSyncTime.entries[partitionIndex].value;
this.batch.vother.col[this.iter] = elem.other;
this.batch.payload.col[this.iter] = elem.payload;
this.batch.key.col[this.iter] = elem.key;
this.batch.hash.col[this.iter] = hash;
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
}
}
this.tentativeOutput.entries[partitionIndex].value.Clear(); // Clear the tentative output list
}
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
}
}
OnLowWatermark(synctime);
}
else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime)
{
var key = srckey[i];
long synctime = src_vsync[i];
if (!this.IsSyncTimeSimultaneityFree)
{
var partitionKey = this.getPartitionKey(key);
int partitionIndex = EnsurePartition(partitionKey);
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput.entries[partitionIndex].value);
foreach (var mapIndex in this.seenPartitions.entries[partitionIndex].value)
{
this.seenEvent.Remove(mapIndex);
}
if (this.tentativeOutput.Count > 0)
{
tentativeVisibleTraverser.currIndex = 0;
while (tentativeVisibleTraverser.Next(out int index, out int hash))
{
var elem = this.tentativeOutput.entries[partitionIndex].value.Values[index];
this.batch.vsync.col[this.iter] = this.lastSyncTime.entries[partitionIndex].value;
this.batch.vother.col[this.iter] = elem.other;
this.batch.payload.col[this.iter] = elem.payload;
this.batch.key.col[this.iter] = elem.key;
this.batch.hash.col[this.iter] = hash;
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
}
}
this.tentativeOutput.entries[partitionIndex].value.Clear(); // Clear the tentative output list
}
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
}
this.batch.vsync.col[this.iter] = synctime;
this.batch.vother.col[this.iter] = long.MinValue;
this.batch.payload.col[this.iter] = default;
this.batch.key.col[this.iter] = key;
this.batch.hash.col[this.iter] = src_hash[i];
this.batch.bitvector.col[this.iter >> 6] |= (1L << (this.iter & 0x3f));
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
}
}
}
}
}
batch.Free();
}
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch)
{
var count = batch.Count;
var srckey = batch.key.col;
fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col)
{
fixed(int *src_hash = batch.hash.col)
{
for (int i = 0; i < count; i++)
{
if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
{
var partitionKey = this.getPartitionKey(srckey[i]);
int partitionIndex = EnsurePartition(partitionKey);
long synctime = src_vsync[i];
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
ProcessCurrentTimestamp(partitionKey, partitionIndex);
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
int keyHeads_index;
bool keyHeadExists = false;
var keyHeadsFindTraverser = new FastMap <TKey> .FindTraverser(this.keyHeads.entries[partitionIndex].value);
if (keyHeadsFindTraverser.Find(src_hash[i]))
{
while (keyHeadsFindTraverser.Next(out keyHeads_index))
{
if (!this.keyEqualityComparer(this.keyHeads.entries[partitionIndex].value.Values[keyHeads_index], srckey[i]))
{
continue;
}
// Found entry, this key has been processed before
keyHeadExists = true;
break;
}
}
if (!keyHeadExists)
{
// Apply new transitions, update existing transitions
bool found = this.activeFindTraverser.Find(src_hash[i]);
if (found)
{
while (this.activeFindTraverser.Next(out int activeFind_index))
{
var state = this.activeStates.Values[activeFind_index];
if (!this.keyEqualityComparer(state.key, srckey[i]))
{
continue;
}
// TODO: Found entry, create and accumulate new tentative transitions from current state
if (state.PatternStartTimestamp + this.MaxDuration > synctime)
{
var currentStateMap = this.multiEventStateMap[state.toState];
if (currentStateMap != null)
{
var m = currentStateMap.Length;
for (int cnt = 0; cnt < m; cnt++)
{
var arcinfo = currentStateMap[cnt];
if (activeFind_index == -1)
{
activeFind_index = this.activeStates.Insert(src_hash[i]);
}
this.activeStates.Values[activeFind_index].arcinfo = arcinfo;
this.activeStates.Values[activeFind_index].key = state.key;
this.activeStates.Values[activeFind_index].fromState = state.toState;
this.activeStates.Values[activeFind_index].toState = arcinfo.toState;
this.activeStates.Values[activeFind_index].PatternStartTimestamp = state.PatternStartTimestamp;
this.activeStates.Values[activeFind_index].register = state.register;
this.activeStates.Values[activeFind_index].accumulator = arcinfo.Initialize(synctime, state.register);
this.activeStates.Values[activeFind_index].accumulator = arcinfo.Accumulate(synctime, batch.payload.col[i], state.register, this.activeStates.Values[activeFind_index].accumulator);
activeFind_index = -1;
}
}
}
// Remove current state
if (activeFind_index != -1)
{
this.activeFindTraverser.Remove();
}
}
}
// Insert & accumulate new tentative transitions from start state
for (int counter = 0; counter < this.numStartStates; counter++)
{
int startState = this.startStates[counter];
var startStateMap = this.multiEventStateMap[startState];
var m = startStateMap.Length;
for (int cnt = 0; cnt < m; cnt++)
{
var arcinfo = startStateMap[cnt];
int index = this.activeFindTraverser.InsertAt(); // have to ensure the new states go to the end of the list
this.activeStates.Values[index].arcinfo = arcinfo;
this.activeStates.Values[index].key = srckey[i];
this.activeStates.Values[index].fromState = startState;
this.activeStates.Values[index].toState = arcinfo.toState;
this.activeStates.Values[index].PatternStartTimestamp = synctime;
this.activeStates.Values[index].register = this.defaultRegister;
this.activeStates.Values[index].accumulator = arcinfo.Initialize(synctime, this.defaultRegister);
this.activeStates.Values[index].accumulator = arcinfo.Accumulate(synctime, batch.payload.col[i], this.defaultRegister, this.activeStates.Values[index].accumulator);
}
}
// Update keyHeads to indicate that this key has been inserted
keyHeads_index = this.keyHeads.entries[partitionIndex].value.Insert(src_hash[i]);
this.keyHeads.entries[partitionIndex].value.Values[keyHeads_index] = srckey[i];
// Done processing this event
continue;
}
// Not the first insert of this key for this timestamp, perform accumulate for all tentative states
if (this.activeFindTraverser.Find(src_hash[i]))
{
while (this.activeFindTraverser.Next(out int activeFind_index))
{
var state2 = this.activeStates.Values[activeFind_index];
if (!this.keyEqualityComparer(state2.key, srckey[i]))
{
continue;
}
// Found tentative entry, accumulate
this.activeStates.Values[activeFind_index].accumulator = state2.arcinfo.Accumulate(synctime, batch.payload.col[i], state2.register, state2.accumulator);
}
}
}
else if (src_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime)
{
int partitionIndex = FastDictionary2 <TPartitionKey, List <TKey> > .IteratorStart;
long synctime = src_vsync[i];
while (this.lastSyncTime.Iterate(ref partitionIndex))
{
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
ProcessCurrentTimestamp(this.lastSyncTime.entries[partitionIndex].key, partitionIndex);
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
}
OnLowWatermark(synctime);
}
else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime)
{
var partitionKey = this.getPartitionKey(srckey[i]);
int partitionIndex = EnsurePartition(partitionKey);
long synctime = src_vsync[i];
if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
{
ProcessCurrentTimestamp(partitionKey, partitionIndex);
this.lastSyncTime.entries[partitionIndex].value = synctime;
}
}
}
}
}
batch.Free();
}
private static void SimpleMapTest()
{
var map = new FastMap <string>();
Assert.IsTrue(map.IsEmpty);
Assert.AreEqual(0, map.Count);
Assert.IsFalse(map.Find(5).Next(out _));
Assert.IsFalse(map.Find(6).Next(out _));
Assert.IsFalse(map.Find(7).Next(out _));
int indexA5 = map.Insert(5, "a");
Assert.IsFalse(map.IsEmpty);
Assert.AreEqual(1, map.Count);
Assert.IsTrue(map.Find(5).Next(out var index));
Assert.AreEqual(indexA5, index);
Assert.IsFalse(map.Find(6).Next(out _));
Assert.IsFalse(map.Find(7).Next(out _));
int indexB6 = map.Insert(6, "b");
int indexA5Two = map.Insert(5, "a");
int indexB5 = map.Insert(5, "b");
Assert.IsTrue(indexA5 != indexA5Two);
Assert.IsTrue(indexA5 != indexB5);
Assert.IsTrue(indexA5Two != indexB5);
Assert.IsFalse(map.IsEmpty);
Assert.AreEqual(4, map.Count);
var traverser = map.Find(5);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexB5, index);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexA5Two, index);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexA5, index);
Assert.IsFalse(traverser.Next(out _));
traverser = map.Find(6);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexB6, index);
Assert.IsFalse(traverser.Next(out _));
Assert.IsFalse(map.Find(7).Next(out _));
map.Remove(indexA5);
Assert.IsFalse(map.IsEmpty);
Assert.AreEqual(3, map.Count);
traverser = map.Find(5);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexB5, index);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexA5Two, index);
Assert.IsFalse(traverser.Next(out _));
traverser = map.Find(6);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexB6, index);
Assert.IsFalse(traverser.Next(out _));
Assert.IsFalse(map.Find(7).Next(out _));
map.Remove(indexA5Two);
map.Remove(indexB6);
Assert.IsFalse(map.IsEmpty);
Assert.AreEqual(1, map.Count);
traverser = map.Find(5);
Assert.IsTrue(traverser.Next(out index));
Assert.AreEqual(indexB5, index);
Assert.IsFalse(traverser.Next(out _));
Assert.IsFalse(map.Find(6).Next(out _));
Assert.IsFalse(map.Find(7).Next(out _));
map.Remove(indexB5);
Assert.IsTrue(map.IsEmpty);
Assert.AreEqual(0, map.Count);
Assert.IsFalse(map.Find(5).Next(out _));
Assert.IsFalse(map.Find(6).Next(out _));
Assert.IsFalse(map.Find(7).Next(out _));
}
private static void LargerTest()
{
var map = new FastMap <string>(1);
const int Size = 100;
for (int i = 0; i < Size; i++)
{
map.Insert(i, "one" + i);
map.Insert(i, "two" + i);
}
for (int i = 0; i < Size; i++)
{
map.Insert(i, "three" + i);
}
Assert.AreEqual(Size * 3, map.Count);
for (int i = 0; i < Size; i++)
{
var traverse = map.Find(i);
bool hasOne = false;
bool hasTwo = false;
bool hasThree = false;
for (int j = 0; j < 3; j++)
{
Assert.IsTrue(traverse.Next(out var index));
if (map.Values[index] == "one" + i)
{
hasOne = true;
}
else if (map.Values[index] == "two" + i)
{
hasTwo = true;
}
else if (map.Values[index] == "three" + i)
{
hasThree = true;
}
else
{
Assert.Fail();
}
}
Assert.IsFalse(traverse.Next(out _));
Assert.IsTrue(hasOne);
Assert.IsTrue(hasTwo);
Assert.IsTrue(hasThree);
}
for (int i = 0; i < Size; i++)
{
var traverse = map.Find(i);
while (traverse.Next(out int index))
{
if (map.Values[index] == "two" + i)
{
map.Remove(index);
break;
}
}
}
Assert.AreEqual(Size * 2, map.Count);
for (int i = 0; i < Size; i++)
{
var traverse = map.Find(i);
bool hasOne = false;
bool hasThree = false;
for (int j = 0; j < 2; j++)
{
Assert.IsTrue(traverse.Next(out var index));
if (map.Values[index] == "one" + i)
{
hasOne = true;
}
else if (map.Values[index] == "three" + i)
{
hasThree = true;
}
else
{
Assert.Fail();
}
}
Assert.IsFalse(traverse.Next(out _));
Assert.IsTrue(hasOne);
Assert.IsTrue(hasThree);
}
for (int i = 0; i < Size; i++)
{
var traverse = map.Find(i);
while (traverse.Next(out int index))
{
if (map.Values[index] == "one" + i)
{
map.Remove(index);
break;
}
}
}
Assert.AreEqual(Size, map.Count);
for (int i = 0; i < Size; i++)
{
var traverse = map.Find(i);
Assert.IsTrue(traverse.Next(out int index));
Assert.AreEqual("three" + i, map.Values[index]);
Assert.IsFalse(traverse.Next(out _));
}
for (int i = 0; i < Size; i++)
{
var traverse = map.Find(i);
while (traverse.Next(out int index))
{
if (map.Values[index] == "three" + i)
{
map.Remove(index);
break;
}
}
}
Assert.IsTrue(map.IsEmpty);
}
private void ProcessPendingEntries()
{
foreach (var pKey in this.processQueue)
{
PooledElasticCircularBuffer <LEntry> leftWorking = null;
PooledElasticCircularBuffer <REntry> rightWorking = null;
this.leftQueue.Lookup(pKey, out int index);
leftWorking = this.leftQueue.entries[index].value;
rightWorking = this.rightQueue.entries[index].value;
FastMap <ActiveEvent <TLeft> > leftEdgeMapForPartition = null;
FastMap <ActiveEvent <TRight> > rightEdgeMapForPartition = null;
this.partitionData.Lookup(pKey, out index);
var partition = this.partitionData.entries[index].value;
leftEdgeMapForPartition = partition.leftEdgeMap;
rightEdgeMapForPartition = partition.rightEdgeMap;
while (true)
{
bool hasLeftBatch = leftWorking.TryPeekFirst(out LEntry leftEntry);
bool hasRightBatch = rightWorking.TryPeekFirst(out REntry rightEntry);
FastMap <ActiveEvent <TRight> > .FindTraverser rightEdges = default;
FastMap <ActiveEvent <TLeft> > .FindTraverser leftEdges = default;
if (hasLeftBatch && hasRightBatch)
{
UpdateNextLeftTime(partition, leftEntry.Sync);
UpdateNextRightTime(partition, rightEntry.Sync);
if (partition.nextLeftTime <= partition.nextRightTime)
{
if (leftEntry.Other != long.MinValue)
{
TKey key = leftEntry.Key;
var hash = leftEntry.Hash;
if (rightEdgeMapForPartition.Find(hash, ref rightEdges))
{
while (rightEdges.Next(out int rightIndex))
{
if (this.keyComparer(key, rightEdgeMapForPartition.Values[rightIndex].Key))
{
OutputStartEdge(partition.nextLeftTime, ref key, ref leftEntry.Payload, ref rightEdgeMapForPartition.Values[rightIndex].Payload, hash);
}
}
}
if (!partition.isRightComplete)
{
int newIndex = leftEdgeMapForPartition.Insert(hash);
leftEdgeMapForPartition.Values[newIndex].Populate(ref key, ref leftEntry.Payload);
}
UpdateNextLeftTime(partition, leftEntry.Sync);
}
else
{
OutputPunctuation(leftEntry.Sync, ref leftEntry.Key, leftEntry.Hash);
}
}
else
{
if (rightEntry.Other != long.MinValue)
{
TKey key = rightEntry.Key;
var hash = rightEntry.Hash;
if (leftEdgeMapForPartition.Find(hash, ref leftEdges))
{
while (leftEdges.Next(out int leftIndex))
{
if (this.keyComparer(key, leftEdgeMapForPartition.Values[leftIndex].Key))
{
OutputStartEdge(partition.nextRightTime, ref key, ref leftEdgeMapForPartition.Values[leftIndex].Payload, ref rightEntry.Payload, hash);
}
}
}
if (!partition.isLeftComplete)
{
int newIndex = rightEdgeMapForPartition.Insert(hash);
rightEdgeMapForPartition.Values[newIndex].Populate(ref key, ref rightEntry.Payload);
}
UpdateNextRightTime(partition, rightEntry.Sync);
}
else
{
OutputPunctuation(rightEntry.Sync, ref rightEntry.Key, rightEntry.Hash);
}
}
}
else if (hasLeftBatch)
{
UpdateNextLeftTime(partition, leftEntry.Sync);
if (leftEntry.Other != long.MinValue)
{
TKey key = leftEntry.Key;
var hash = leftEntry.Hash;
if (rightEdgeMapForPartition.Find(hash, ref rightEdges))
{
while (rightEdges.Next(out int rightIndex))
{
if (this.keyComparer(key, rightEdgeMapForPartition.Values[rightIndex].Key))
{
OutputStartEdge(partition.nextLeftTime, ref key, ref leftEntry.Payload, ref rightEdgeMapForPartition.Values[rightIndex].Payload, hash);
}
}
}
if (!partition.isRightComplete)
{
int newIndex = leftEdgeMapForPartition.Insert(hash);
leftEdgeMapForPartition.Values[newIndex].Populate(ref key, ref leftEntry.Payload);
}
UpdateNextLeftTime(partition, leftEntry.Sync);
}
else
{
OutputPunctuation(leftEntry.Sync, ref leftEntry.Key, leftEntry.Hash);
return;
}
}
else if (hasRightBatch)
{
UpdateNextRightTime(partition, rightEntry.Sync);
if (rightEntry.Other != long.MinValue)
{
TKey key = rightEntry.Key;
var hash = rightEntry.Hash;
if (leftEdgeMapForPartition.Find(hash, ref leftEdges))
{
while (leftEdges.Next(out int leftIndex))
{
if (this.keyComparer(key, leftEdgeMapForPartition.Values[leftIndex].Key))
{
OutputStartEdge(partition.nextRightTime, ref key, ref leftEdgeMapForPartition.Values[leftIndex].Payload, ref rightEntry.Payload, hash);
}
}
}
if (!partition.isLeftComplete)
{
int newIndex = rightEdgeMapForPartition.Insert(hash);
rightEdgeMapForPartition.Values[newIndex].Populate(ref key, ref rightEntry.Payload);
}
UpdateNextRightTime(partition, rightEntry.Sync);
}
else
{
OutputPunctuation(rightEntry.Sync, ref rightEntry.Key, rightEntry.Hash);
}
}
else
{
if (partition.nextLeftTime < this.lastLeftCTI)
{
UpdateNextLeftTime(partition, this.lastLeftCTI);
}
if (partition.nextRightTime < this.lastRightCTI)
{
UpdateNextRightTime(partition, this.lastRightCTI);
}
this.cleanKeys.Add(pKey);
break;
}
}
}
if (this.emitCTI)
{
var earliest = Math.Min(this.lastLeftCTI, this.lastRightCTI);
AddLowWatermarkToBatch(earliest);
this.emitCTI = false;
foreach (var p in this.cleanKeys)
{
this.leftQueue.Lookup(p, out int index);
var l = this.leftQueue.entries[index];
var r = this.rightQueue.entries[index];
if (l.value.Count == 0 && r.value.Count == 0)
{
this.seenKeys.Remove(p);
l.value.Dispose();
this.leftQueue.Remove(p);
r.value.Dispose();
this.rightQueue.Remove(p);
}
}
this.cleanKeys.Clear();
}
this.processQueue.Clear();
}
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch)
{
var stack = new Stack <int>();
var activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates);
var tentativeFindTraverser = new FastMap <OutputEvent <TKey, TRegister> > .FindTraverser(this.tentativeOutput);
var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput);
var count = batch.Count;
var dest_vsync = this.batch.vsync.col;
var dest_vother = this.batch.vother.col;
var destkey = this.batch.key.col;
var dest_hash = this.batch.hash.col;
var srckey = batch.key.col;
fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col)
{
fixed(int *src_hash = batch.hash.col)
{
for (int i = 0; i < count; i++)
{
if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
{
long synctime = src_vsync[i];
if (!this.IsSyncTimeSimultaneityFree)
{
int index;
if (synctime > this.lastSyncTime) // move time forward
{
this.seenEvent.Clear();
if (this.tentativeOutput.Count > 0)
{
tentativeVisibleTraverser.currIndex = 0;
while (tentativeVisibleTraverser.Next(out index, out int hash))
{
var elem = this.tentativeOutput.Values[index];
dest_vsync[this.iter] = this.lastSyncTime;
dest_vother[this.iter] = elem.other;
this.batch.payload.col[this.iter] = elem.payload;
destkey[this.iter] = elem.key;
dest_hash[this.iter] = hash;
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
this.tentativeOutput.Clear(); // Clear the tentative output list
}
this.lastSyncTime = synctime;
}
if (this.seenEvent.Lookup(srckey[i], out index)) // Incoming event is a simultaneous one
{
if (this.seenEvent.entries[index].value == 1) // Detecting first duplicate, need to adjust state
{
this.seenEvent.entries[index].value = 2;
// Delete tentative output for that key
if (tentativeFindTraverser.Find(src_hash[i]))
{
while (tentativeFindTraverser.Next(out index))
{
if (this.keyEqualityComparer(this.tentativeOutput.Values[index].key, srckey[i]))
{
tentativeFindTraverser.Remove();
}
}
}
// Delete active states for that key
if (activeFindTraverser.Find(src_hash[i]))
{
while (activeFindTraverser.Next(out index))
{
if (this.keyEqualityComparer(this.activeStates.Values[index].key, srckey[i]))
{
activeFindTraverser.Remove();
}
}
}
}
// Dont process this event
continue;
}
else
{
this.seenEvent.Insert(ref index, srckey[i], 1);
}
}
/* (1) Process currently active states */
bool ended = true;
if (activeFindTraverser.Find(src_hash[i]))
{
int orig_index;
// Track which active states need to be inserted after the current traversal
var newActiveStates = new List <GroupedActiveState <TKey, TRegister> >();
while (activeFindTraverser.Next(out int index))
{
orig_index = index;
var state = this.activeStates.Values[index];
if (!this.keyEqualityComparer(state.key, srckey[i]))
{
continue;
}
if (state.PatternStartTimestamp + this.MaxDuration > synctime)
{
var currentStateMap = this.singleEventStateMap[state.state];
if (currentStateMap != null)
{
var m = currentStateMap.Length;
for (int cnt = 0; cnt < m; cnt++)
{
var arcinfo = currentStateMap[cnt];
if (arcinfo.Fence(synctime, batch[i], state.register))
{
var newReg = arcinfo.Transfer == null
? state.register
: arcinfo.Transfer(synctime, batch[i], state.register);
int ns = arcinfo.toState;
while (true)
{
if (this.isFinal[ns])
{
var otherTime = Math.Min(state.PatternStartTimestamp + this.MaxDuration, StreamEvent.InfinitySyncTime);
if (!this.IsSyncTimeSimultaneityFree)
{
int ind = this.tentativeOutput.Insert(src_hash[i]);
this.tentativeOutput.Values[ind].other = otherTime;
this.tentativeOutput.Values[ind].key = srckey[i];
this.tentativeOutput.Values[ind].payload = newReg;
}
else
{
dest_vsync[this.iter] = synctime;
dest_vother[this.iter] = otherTime;
this.batch[this.iter] = newReg;
destkey[this.iter] = srckey[i];
dest_hash[this.iter] = src_hash[i];
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
}
if (this.hasOutgoingArcs[ns])
{
// Since we will eventually remove this state/index from activeStates, attempt to reuse this index for the outgoing state instead of deleting/re-adding
// If index is already -1, this means we've already reused the state and must allocate/insert a new index for the outgoing state.
if (index != -1)
{
this.activeStates.Values[index].key = srckey[i];
this.activeStates.Values[index].state = ns;
this.activeStates.Values[index].register = newReg;
this.activeStates.Values[index].PatternStartTimestamp = state.PatternStartTimestamp;
index = -1;
}
else
{
// Do not attempt to insert directly into activeStates, as that could corrupt the traversal state.
newActiveStates.Add(new GroupedActiveState <TKey, TRegister>
{
key = srckey[i],
state = ns,
register = newReg,
PatternStartTimestamp = state.PatternStartTimestamp,
});
}
ended = false;
// Add epsilon arc destinations to stack
if (this.epsilonStateMap == null)
{
break;
}
if (this.epsilonStateMap[ns] != null)
{
for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++)
{
stack.Push(this.epsilonStateMap[ns][cnt2]);
}
}
}
if (stack.Count == 0)
{
break;
}
ns = stack.Pop();
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one successful transition
}
}
}
}
}
if (index == orig_index)
{
activeFindTraverser.Remove();
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one active state
}
}
// Now that we are done traversing the current active states, add any new ones.
foreach (var newActiveState in newActiveStates)
{
this.activeStates.Insert(src_hash[i], newActiveState);
}
}
/* (2) Start new activations from the start state(s) */
if (!this.AllowOverlappingInstances && !ended)
{
continue;
}
for (int counter = 0; counter < this.numStartStates; counter++)
{
int startState = this.startStates[counter];
var startStateMap = this.singleEventStateMap[startState];
if (startStateMap != null)
{
var m = startStateMap.Length;
for (int cnt = 0; cnt < m; cnt++)
{
var arcinfo = startStateMap[cnt];
if (arcinfo.Fence(synctime, batch[i], this.defaultRegister))
{
var newReg = arcinfo.Transfer == null
? this.defaultRegister
: arcinfo.Transfer(synctime, batch[i], this.defaultRegister);
int ns = arcinfo.toState;
while (true)
{
if (this.isFinal[ns])
{
var otherTime = Math.Min(synctime + this.MaxDuration, StreamEvent.InfinitySyncTime);
if (!this.IsSyncTimeSimultaneityFree)
{
int ind = this.tentativeOutput.Insert(src_hash[i]);
this.tentativeOutput.Values[ind].other = otherTime;
this.tentativeOutput.Values[ind].key = srckey[i];
this.tentativeOutput.Values[ind].payload = newReg;
}
else
{
dest_vsync[this.iter] = synctime;
dest_vother[this.iter] = otherTime;
this.batch[this.iter] = newReg;
destkey[this.iter] = srckey[i];
dest_hash[this.iter] = src_hash[i];
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
}
if (this.hasOutgoingArcs[ns])
{
int index = this.activeStates.Insert(src_hash[i]);
this.activeStates.Values[index].key = srckey[i];
this.activeStates.Values[index].state = ns;
this.activeStates.Values[index].register = newReg;
this.activeStates.Values[index].PatternStartTimestamp = synctime;
// Add epsilon arc destinations to stack
if (this.epsilonStateMap == null)
{
break;
}
if (this.epsilonStateMap[ns] != null)
{
for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++)
{
stack.Push(this.epsilonStateMap[ns][cnt2]);
}
}
}
if (stack.Count == 0)
{
break;
}
ns = stack.Pop();
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one successful transition
}
}
}
}
if (this.IsDeterministic)
{
break; // We are guaranteed to have only one start state
}
}
}
else if (batch.vother.col[i] < 0)
{
long synctime = src_vsync[i];
if (!this.IsSyncTimeSimultaneityFree && synctime > this.lastSyncTime) // move time forward
{
this.seenEvent.Clear();
if (this.tentativeOutput.Count > 0)
{
tentativeVisibleTraverser.currIndex = 0;
while (tentativeVisibleTraverser.Next(out int index, out int hash))
{
var elem = this.tentativeOutput.Values[index];
this.batch.vsync.col[this.iter] = this.lastSyncTime;
this.batch.vother.col[this.iter] = elem.other;
this.batch.payload.col[this.iter] = elem.payload;
this.batch.key.col[this.iter] = elem.key;
this.batch.hash.col[this.iter] = hash;
this.iter++;
if (this.iter == Config.DataBatchSize)
{
FlushContents();
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
this.tentativeOutput.Clear(); // Clear the tentative output list
}
this.lastSyncTime = synctime;
}
// Update dest_* on punctuation in case this event will hit the batch boundary and allocate a new batch
OnPunctuation(synctime);
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
}
}
}
batch.Free();
}
