public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; batch.bitvector = batch.bitvector.MakeWritable(this.pool.bitvectorPool); fixed(long *vsync = batch.vsync.col) fixed(long *vother = batch.vother.col) fixed(long *bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0 || vother[i] == long.MinValue) { if (batch.vother.col[i] == long.MinValue) // Punctuation { if (vsync[i] == StreamEvent.InfinitySyncTime) { OutputAllEvents(); } else { OutputCompletedIntervals(); } this.lastCti = Math.Max(vsync[i], this.lastCti); this.lastSyncTime = Math.Max(vsync[i], this.lastSyncTime); AddPunctuationToBatch(batch.vsync.col[i]); } else if (vsync[i] < vother[i]) // Start edge or interval { var evt = new ActiveEvent { End = vother[i], Payload = batch[i], Key = batch.key.col[i], Hash = batch.hash.col[i] }; if (!this.eventMap.TryGetValue(vsync[i], out var entry)) { this.dictPool.Get(out entry); this.eventMap.Add(vsync[i], entry); } if (!entry.Lookup(evt, out int index)) { entry.Insert(evt, 1); } else { entry.entries[index].value++; } } else // end edge { // lookup corresponding start edge var lookupevt = new ActiveEvent { End = StreamEvent.InfinitySyncTime, Payload = batch[i], Key = batch.key.col[i], Hash = batch.hash.col[i] }; if (!this.eventMap.TryGetValue(vother[i], out FastDictionary2 <ActiveEvent, int> entry)) { throw new InvalidOperationException("Found end edge without corresponding start edge"); } if (!entry.Lookup(lookupevt, out int index)) { throw new InvalidOperationException("Found end edge without corresponding start edge"); } // Set interval payload to the payload of the original start-edge // (in case they are different due to an optimized payload equality comparer) lookupevt.Payload = entry.entries[index].key.Payload; // delete the start edge entry.entries[index].value--; if (entry.entries[index].value == 0) { entry.Remove(lookupevt); } // insert interval lookupevt.End = batch.vsync.col[i]; if (!entry.Lookup(lookupevt, out index)) { entry.Insert(lookupevt, 1); } else { entry.entries[index].value++; } OutputCompletedIntervals(); // Can make this more efficient by trying only if the first event in index got completed } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; var srckey = batch.key.col; SavedEventList <TKey, TPayload> sevref = default; fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col) { fixed(int *src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0) { var partitionKey = this.getPartitionKey(srckey[i]); int partitionIndex = EnsurePartition(partitionKey); long synctime = src_vsync[i]; int index; if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { ProcessCurrentTimestamp(partitionKey, partitionIndex); this.lastSyncTime.entries[partitionIndex].value = synctime; } bool done = false; var eventListTraverser = new FastMap <SavedEventList <TKey, TPayload> > .FindTraverser(this.currentTimestampEventList.entries[partitionIndex].value); if (eventListTraverser.Find(src_hash[i])) { while (eventListTraverser.Next(out index)) { var state = this.currentTimestampEventList.entries[partitionIndex].value.Values[index]; if (this.keyEqualityComparer(state.key, srckey[i])) { state.payloads.Add(batch.payload.col[i]); done = true; break; } } } if (!done) { index = this.currentTimestampEventList.entries[partitionIndex].value.Insert(src_hash[i]); sevref.payloads = new List <TPayload>(10) { batch.payload.col[i] }; sevref.key = srckey[i]; this.currentTimestampEventList.entries[partitionIndex].value.Values[index] = sevref; } } else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { int partitionIndex = FastDictionary2 <TPartitionKey, List <TKey> > .IteratorStart; long synctime = src_vsync[i]; while (this.lastSyncTime.Iterate(ref partitionIndex)) { if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { ProcessCurrentTimestamp(this.lastSyncTime.entries[partitionIndex].key, partitionIndex); this.lastSyncTime.entries[partitionIndex].value = synctime; } } OnLowWatermark(synctime); } else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { var partitionKey = this.getPartitionKey(srckey[i]); int partitionIndex = EnsurePartition(partitionKey); long synctime = src_vsync[i]; if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { ProcessCurrentTimestamp(partitionKey, partitionIndex); this.lastSyncTime.entries[partitionIndex].value = synctime; } } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; fixed(long *bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { if (batch.vsync.col[i] >= StreamEvent.MinSyncTime + this.duration) { ReachTime(batch.vsync.col[i] - this.duration); } if (batch.vother.col[i] == StreamEvent.InfinitySyncTime) { // For start events, we queue them up // until we are [duration] time past to make sure that // no end edges have been moved earlier int index = this.syncTimeMap.Insert(batch.hash.col[i]); this.syncTimeMap.Values[index].Populate(batch.key.col[i], batch[i], batch.vsync.col[i], StreamEvent.InfinitySyncTime, batch.hash.col[i]); this.endPointHeap.Insert(batch.vsync.col[i], index); } else if (batch.vother.col[i] > batch.vsync.col[i]) { // For intervals, we clip the duration as well before queueing // Events with durations of zero or less are dropped var sync = batch.vsync.col[i]; var other = batch.vother.col[i] - this.duration; if (other <= sync) { continue; } int index = this.syncTimeMap.Insert(batch.hash.col[i]); this.syncTimeMap.Values[index].Populate(batch.key.col[i], batch[i], sync, other, batch.hash.col[i]); this.endPointHeap.Insert(batch.vsync.col[i], index); } else { var sync = batch.vsync.col[i] - this.duration; var other = batch.vother.col[i]; if (other >= sync) { // If we have a contracted event, do not add the end edge to the batch. // Also, add the payload to a list of events to be purged. this.contractedToZero.Add(other, new ActiveEvent { Key = batch.key.col[i], Payload = batch[i], Sync = sync, Other = other, Hash = batch.hash.col[i] }); } else { int ind = this.output.Count++; this.output.vsync.col[ind] = sync; this.output.vother.col[ind] = other; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } else if (batch.vother.col[i] == long.MinValue) { long syncTime = (batch.vsync.col[i] == StreamEvent.InfinitySyncTime ? StreamEvent.InfinitySyncTime : batch.vsync.col[i] - this.duration); ReachTime(syncTime); int ind = this.output.Count++; this.output.vsync.col[ind] = syncTime; this.output.vother.col[ind] = long.MinValue; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = default; this.output.hash.col[ind] = batch.hash.col[i]; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; var srckey = batch.key.col; fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col) { fixed(int *src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0) { var partitionKey = this.getPartitionKey(srckey[i]); int partitionIndex = EnsurePartition(partitionKey); long synctime = src_vsync[i]; if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { ProcessCurrentTimestamp(partitionIndex); this.lastSyncTime.entries[partitionIndex].value = synctime; } int keyHeads_index; bool keyHeadExists = false; var keyHeadsFindTraverser = new FastMap <TKey> .FindTraverser(this.keyHeads.entries[partitionIndex].value); if (keyHeadsFindTraverser.Find(src_hash[i])) { while (keyHeadsFindTraverser.Next(out keyHeads_index)) { if (!this.keyEqualityComparer(this.keyHeads.entries[partitionIndex].value.Values[keyHeads_index], srckey[i])) { continue; } // Found entry, this key has been processed before keyHeadExists = true; break; } } if (!keyHeadExists) { // Apply new transitions, update existing transitions bool found = this.activeFindTraverser.Find(src_hash[i]); if (found) { while (this.activeFindTraverser.Next(out int activeFind_index)) { var state = this.activeStates.Values[activeFind_index]; if (!this.keyEqualityComparer(state.key, srckey[i])) { continue; } // TODO: Found entry, create and accumulate new tentative transitions from current state if (state.PatternStartTimestamp + this.MaxDuration > synctime) { var currentStateMap = this.multiEventStateMap[state.toState]; if (currentStateMap != null) { var m = currentStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = currentStateMap[cnt]; if (activeFind_index == -1) { activeFind_index = this.activeStates.Insert(src_hash[i]); } this.activeStates.Values[activeFind_index].arcinfo = arcinfo; this.activeStates.Values[activeFind_index].key = state.key; this.activeStates.Values[activeFind_index].fromState = state.toState; this.activeStates.Values[activeFind_index].toState = arcinfo.toState; this.activeStates.Values[activeFind_index].PatternStartTimestamp = state.PatternStartTimestamp; this.activeStates.Values[activeFind_index].register = state.register; this.activeStates.Values[activeFind_index].accumulator = arcinfo.Initialize(synctime, state.register); this.activeStates.Values[activeFind_index].accumulator = arcinfo.Accumulate(synctime, batch.payload.col[i], state.register, this.activeStates.Values[activeFind_index].accumulator); activeFind_index = -1; } } } // Remove current state if (activeFind_index != -1) { this.activeFindTraverser.Remove(); } } } // Insert & accumulate new tentative transitions from start state for (int counter = 0; counter < this.numStartStates; counter++) { int startState = this.startStates[counter]; var startStateMap = this.multiEventStateMap[startState]; var m = startStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = startStateMap[cnt]; int index = this.activeFindTraverser.InsertAt(); // have to ensure the new states go to the end of the list this.activeStates.Values[index].arcinfo = arcinfo; this.activeStates.Values[index].key = srckey[i]; this.activeStates.Values[index].fromState = startState; this.activeStates.Values[index].toState = arcinfo.toState; this.activeStates.Values[index].PatternStartTimestamp = synctime; this.activeStates.Values[index].register = this.defaultRegister; this.activeStates.Values[index].accumulator = arcinfo.Initialize(synctime, this.defaultRegister); this.activeStates.Values[index].accumulator = arcinfo.Accumulate(synctime, batch.payload.col[i], this.defaultRegister, this.activeStates.Values[index].accumulator); } } // Update keyHeads to indicate that this key has been inserted keyHeads_index = this.keyHeads.entries[partitionIndex].value.Insert(src_hash[i]); this.keyHeads.entries[partitionIndex].value.Values[keyHeads_index] = srckey[i]; // Done processing this event continue; } // Not the first insert of this key for this timestamp, perform accumulate for all tentative states if (this.activeFindTraverser.Find(src_hash[i])) { while (this.activeFindTraverser.Next(out int activeFind_index)) { var state2 = this.activeStates.Values[activeFind_index]; if (!this.keyEqualityComparer(state2.key, srckey[i])) { continue; } // Found tentative entry, accumulate this.activeStates.Values[activeFind_index].accumulator = state2.arcinfo.Accumulate(synctime, batch.payload.col[i], state2.register, state2.accumulator); } } } else if (src_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { int partitionIndex = FastDictionary2 <TPartitionKey, List <TKey> > .IteratorStart; long synctime = src_vsync[i]; while (this.lastSyncTime.Iterate(ref partitionIndex)) { if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { ProcessCurrentTimestamp(partitionIndex); this.lastSyncTime.entries[partitionIndex].value = synctime; } } OnLowWatermark(synctime); } else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { var partitionKey = this.getPartitionKey(srckey[i]); int partitionIndex = EnsurePartition(partitionKey); long synctime = src_vsync[i]; if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { ProcessCurrentTimestamp(partitionIndex); this.lastSyncTime.entries[partitionIndex].value = synctime; } } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; fixed(long *bv = batch.bitvector.col) fixed(long *vsync = batch.vsync.col) fixed(long *vother = batch.vother.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { var partition = this.getPartitionKey(batch.key.col[i]); if (!this.partitionData.Lookup(partition, out int timeIndex)) { timeIndex = AllocatePartition(partition); } else if (batch.vsync.col[i] > this.partitionData.entries[timeIndex].value.lastSyncTime) { ReachTime(batch.vsync.col[i]); } if (batch.vother.col[i] == StreamEvent.InfinitySyncTime) // Start edge { int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i] - ((vsync[i] - this.offset) % this.progress + this.progress) % this.progress; this.output.vother.col[ind] = StreamEvent.InfinitySyncTime; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else if (batch.vother.col[i] > batch.vsync.col[i]) // Interval { int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i] - ((vsync[i] - this.offset) % this.progress + this.progress) % this.progress; var temp = Math.Max(vother[i] + this.skip - 1, vsync[i] + this.width); this.output.vother.col[ind] = temp - ((temp - (this.offset + this.width)) % this.skip + this.skip) % this.skip; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else // End edge { var endPointHeap = this.partitionData.entries[timeIndex].value.endPointHeap; var intervalMap = this.partitionData.entries[timeIndex].value.intervalMap; var temp = Math.Max(vsync[i] + this.skip - 1, vother[i] + this.width); int index = intervalMap.Insert(batch.hash.col[i]); intervalMap.Values[index].Populate(batch.key.col[i], batch[i], batch.hash.col[i], vother[i] - ((vother[i] - this.offset) % this.progress + this.progress) % this.progress); endPointHeap.Insert(temp - ((temp - (this.offset + this.width)) % this.skip + this.skip) % this.skip, index); } } else if (batch.vother.col[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { ReachTime(batch.vsync.col[i]); int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i] - ((vsync[i] - this.offset) % this.progress + this.progress) % this.progress; this.output.vother.col[ind] = PartitionedStreamEvent.LowWatermarkOtherTime; this.output.key.col[ind] = default; this.output[ind] = default; this.output.hash.col[ind] = 0; this.output.bitvector.col[ind >> 6] |= 1L << (ind & 0x3f); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else if (batch.vother.col[i] == PartitionedStreamEvent.PunctuationOtherTime) { var partition = this.getPartitionKey(batch.key.col[i]); if (!this.partitionData.Lookup(partition, out int timeIndex)) { timeIndex = AllocatePartition(partition); } ReachTime(timeIndex, batch.vsync.col[i]); int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i] - ((vsync[i] - this.offset) % this.progress + this.progress) % this.progress; this.output.vother.col[ind] = long.MinValue; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = default; this.output.hash.col[ind] = batch.hash.col[i]; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <Empty, TPayload> batch) { var tentativeFindTraverser = new FastLinkedList <OutputEvent <Empty, TRegister> > .ListTraverser(this.tentativeOutput); var tentativeOutputIndex = 0; var count = batch.Count; var dest_vsync = this.batch.vsync.col; var dest_vother = this.batch.vother.col; var destkey = this.batch.key.col; var dest_hash = this.batch.hash.col; var srckey = batch.key.col; fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col) { fixed(int *src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0) { long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { if (synctime > this.lastSyncTime) // move time forward { this.seenEvent = 0; if (this.tentativeOutput.Count > 0) { tentativeOutputIndex = 0; while (this.tentativeOutput.Iterate(ref tentativeOutputIndex)) { var elem = this.tentativeOutput.Values[tentativeOutputIndex]; dest_vsync[this.iter] = this.lastSyncTime; dest_vother[this.iter] = elem.other; this.batch[this.iter] = elem.payload; dest_hash[this.iter] = 0; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } this.tentativeOutput.Clear(); // Clear the tentative output list } this.lastSyncTime = synctime; } if (this.seenEvent > 0) // Incoming event is a simultaneous one { if (this.seenEvent == 1) // Detecting first duplicate, need to adjust state { this.seenEvent = 2; // Delete tentative output for that key this.tentativeOutput.Clear(); // Delete active states for that key this.activeState_state = -1; } // Dont process this event continue; } else { this.seenEvent = 1; } } /* (1) Process currently active states */ if (this.activeState_state >= 0) { if (this.activeState_PatternStartTimestamp + this.MaxDuration > synctime) { var currentStateMap = this.singleEventStateMap[this.activeState_state]; this.activeState_state = -1; // assume the arc does not fire if (currentStateMap != null) { var m = currentStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = currentStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], this.activeState_register)) { if (arcinfo.Transfer != null) { this.activeState_register = arcinfo.Transfer(synctime, batch[i], this.activeState_register); } this.activeState_state = arcinfo.toState; if (this.isFinal[this.activeState_state]) { if (!this.IsSyncTimeSimultaneityFree) { int ind = this.tentativeOutput.Insert(); this.tentativeOutput.Values[ind].other = this.activeState_PatternStartTimestamp + this.MaxDuration; this.tentativeOutput.Values[ind].key = srckey[i]; this.tentativeOutput.Values[ind].payload = this.activeState_register; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = this.activeState_PatternStartTimestamp + this.MaxDuration; this.batch[this.iter] = this.activeState_register; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[this.activeState_state]) { this.activeState_PatternStartTimestamp = synctime; } else { this.activeState_state = -1; } break; // DFA, so only one arc fires } } } } } /* (2) Start new activations from the start state(s) */ if (this.activeState_state >= 0) { continue; } var startStateMap = this.singleEventStateMap[this.startState]; if (startStateMap != null) { var m = startStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = startStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], this.defaultRegister)) { this.activeState_register = arcinfo.Transfer != null ? arcinfo.Transfer(synctime, batch[i], this.defaultRegister) : this.defaultRegister; this.activeState_state = arcinfo.toState; if (this.isFinal[this.activeState_state]) { if (!this.IsSyncTimeSimultaneityFree) { int ind = this.tentativeOutput.Insert(); this.tentativeOutput.Values[ind].other = synctime + this.MaxDuration; this.tentativeOutput.Values[ind].key = srckey[i]; this.tentativeOutput.Values[ind].payload = this.activeState_register; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = synctime + this.MaxDuration; this.batch[this.iter] = this.activeState_register; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[this.activeState_state]) { this.activeState_PatternStartTimestamp = synctime; } else { this.activeState_state = -1; } break; } } } } else if (batch.vother.col[i] < 0) { long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { if (synctime > this.lastSyncTime) // move time forward { this.seenEvent = 0; if (this.tentativeOutput.Count > 0) { tentativeOutputIndex = 0; while (this.tentativeOutput.Iterate(ref tentativeOutputIndex)) { var elem = this.tentativeOutput.Values[tentativeOutputIndex]; this.batch.vsync.col[this.iter] = this.lastSyncTime; this.batch.vother.col[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; this.batch.hash.col[this.iter] = 0; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); } } this.tentativeOutput.Clear(); // Clear the tentative output list } this.lastSyncTime = synctime; } } OnPunctuation(synctime); } } } } batch.Free(); }
public void OnNext(StreamMessage <TKey, TPayload> batch) { this.on(batch); batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> input) { var count = input.Count; fixed(long *src_bv = input.bitvector.col, src_vsync = input.vsync.col, src_vother = input.vother.col) fixed(int *src_hash = input.hash.col) { long *vsync = src_vsync; long *vother = src_vother; int * hash = src_hash; for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0 || *vother < 0) { var partitionKey = this.getPartitionKey(input.key.col[i]); if (this.ClosedEvents.Lookup(partitionKey, out this.ClosedEventsIndex)) { this.ClosedEvents.Insert(ref this.ClosedEventsIndex, partitionKey, new SortedDictionary <long, FastDictionary2 <KHP, List <ActiveEvent> > >()); } if (this.OpenEvents.Lookup(partitionKey, out this.OpenEventsIndex)) { this.OpenEvents.Insert(ref this.OpenEventsIndex, partitionKey, this.OpenEventsGenerator()); } if (this.now.Lookup(partitionKey, out this.nowIndex)) { this.now.Insert(ref this.nowIndex, partitionKey, StreamEvent.MinSyncTime); } if (this.CurrentTimeOpenEventBufferTime.Lookup(partitionKey, out this.CurrentTimeOpenEventBufferTimeIndex)) { this.CurrentTimeOpenEventBufferTime.Insert(ref this.CurrentTimeOpenEventBufferTimeIndex, partitionKey, StreamEvent.MinSyncTime); } if (this.CurrentTimeOpenEventBuffer.Lookup(partitionKey, out this.CurrentTimeOpenEventBufferIndex)) { this.CurrentTimeOpenEventBuffer.Insert(ref this.CurrentTimeOpenEventBufferIndex, partitionKey, this.CurrentTimeOpenEventBufferGenerator()); } var sync = input.vsync.col[i]; if (this.now.entries[this.nowIndex].value < sync) { this.now.entries[this.nowIndex].value = sync; Purge(this.now.entries[this.nowIndex].value); } if (*vother == StreamEvent.InfinitySyncTime) { ActOnStart(input.payload.col[i], input.key.col[i], *hash, *vsync); } else if (*vother == PartitionedStreamEvent.LowWatermarkOtherTime) { PurgeGlobal(*vsync); this.batch.vsync.col[this.outputCount] = *vsync; this.batch.vother.col[this.outputCount] = *vother; this.batch[this.outputCount] = default; this.batch.key.col[this.outputCount] = default; this.batch.hash.col[this.outputCount] = 0; this.outputCount++; if (this.outputCount == Config.DataBatchSize) { FlushContents(); } } else if (*vother == PartitionedStreamEvent.PunctuationOtherTime) { Purge(*vsync); } else if (*vsync < *vother) { ActOnStart(input.payload.col[i], input.key.col[i], *hash, *vsync); ActOnEnd(input.payload.col[i], input.key.col[i], *hash, *vsync, *vother); } else { ActOnEnd(input.payload.col[i], input.key.col[i], *hash, *vother, *vsync); } } vsync++; vother++; hash++; } } input.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var stack = new Stack <int>(); var count = batch.Count; var dest_vsync = this.batch.vsync.col; var dest_vother = this.batch.vother.col; var destkey = this.batch.key.col; var dest_hash = this.batch.hash.col; var srckey = batch.key.col; var activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates); fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col) { fixed(int *src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0) { var key = srckey[i]; var partitionKey = this.getPartitionKey(key); int partitionIndex = EnsurePartition(partitionKey); long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { int index; var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput.entries[partitionIndex].value); if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { this.seenEvent.Remove(key); if (this.tentativeOutput.Count > 0) { tentativeVisibleTraverser.currIndex = 0; while (tentativeVisibleTraverser.Next(out index, out int hash)) { var elem = this.tentativeOutput.entries[partitionIndex].value.Values[index]; dest_vsync[this.iter] = this.lastSyncTime.entries[partitionIndex].value; dest_vother[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; destkey[this.iter] = elem.key; dest_hash[this.iter] = hash; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } this.tentativeOutput.entries[partitionIndex].value.Clear(); // Clear the tentative output list } this.lastSyncTime.entries[partitionIndex].value = synctime; } if (this.seenEvent.Lookup(srckey[i], out index)) // Incoming event is a simultaneous one { if (this.seenEvent.entries[index].value == 1) // Detecting first duplicate, need to adjust state { this.seenEvent.entries[index].value = 2; // Delete tentative output for that key if (!this.IsSyncTimeSimultaneityFree) { var tentativeFindTraverser = new FastMap <OutputEvent <TKey, TRegister> > .FindTraverser(this.tentativeOutput.entries[partitionIndex].value); if (tentativeFindTraverser.Find(src_hash[i])) { while (tentativeFindTraverser.Next(out index)) { if (this.keyEqualityComparer(this.tentativeOutput.entries[partitionIndex].value.Values[index].key, srckey[i])) { tentativeFindTraverser.Remove(); } } } } // Delete active states for that key if (activeFindTraverser.Find(src_hash[i])) { while (activeFindTraverser.Next(out index)) { if (this.keyEqualityComparer(this.activeStates.Values[index].key, srckey[i])) { activeFindTraverser.Remove(); } } } } // Dont process this event continue; } else { this.seenEvent.Insert(srckey[i], 1); } } /* (1) Process currently active states */ bool ended = true; if (activeFindTraverser.Find(src_hash[i])) { int orig_index; // Track which active states need to be inserted after the current traversal var newActiveStates = new List <GroupedActiveState <TKey, TRegister> >(); while (activeFindTraverser.Next(out int index)) { orig_index = index; var state = this.activeStates.Values[index]; if (!this.keyEqualityComparer(state.key, srckey[i])) { continue; } if (state.PatternStartTimestamp + this.MaxDuration > synctime) { var currentStateMap = this.singleEventStateMap[state.state]; if (currentStateMap != null) { var m = currentStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = currentStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], state.register)) { var newReg = arcinfo.Transfer == null ? state.register : arcinfo.Transfer(synctime, batch[i], state.register); int ns = arcinfo.toState; while (true) { if (this.isFinal[ns]) { var otherTime = Math.Min(state.PatternStartTimestamp + this.MaxDuration, StreamEvent.InfinitySyncTime); if (!this.IsSyncTimeSimultaneityFree) { var tentativeOutputEntry = this.tentativeOutput.entries[partitionIndex].value; int ind = tentativeOutputEntry.Insert(src_hash[i]); tentativeOutputEntry.Values[ind].other = otherTime; tentativeOutputEntry.Values[ind].key = srckey[i]; tentativeOutputEntry.Values[ind].payload = newReg; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = otherTime; this.batch[this.iter] = newReg; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[ns]) { // Since we will eventually remove this state/index from activeStates, attempt to reuse this index for the outgoing state instead of deleting/re-adding // If index is already -1, this means we've already reused the state and must allocate/insert a new index for the outgoing state. if (index != -1) { this.activeStates.Values[index].key = srckey[i]; this.activeStates.Values[index].state = ns; this.activeStates.Values[index].register = newReg; this.activeStates.Values[index].PatternStartTimestamp = state.PatternStartTimestamp; index = -1; } else { // Do not attempt to insert directly into activeStates, as that could corrupt the traversal state. newActiveStates.Add(new GroupedActiveState <TKey, TRegister> { key = srckey[i], state = ns, register = newReg, PatternStartTimestamp = state.PatternStartTimestamp, }); } ended = false; // Add epsilon arc destinations to stack if (this.epsilonStateMap == null) { break; } if (this.epsilonStateMap[ns] != null) { for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++) { stack.Push(this.epsilonStateMap[ns][cnt2]); } } } if (stack.Count == 0) { break; } ns = stack.Pop(); } if (this.IsDeterministic) { break; // We are guaranteed to have only one successful transition } } } } } if (index == orig_index) { activeFindTraverser.Remove(); } if (this.IsDeterministic) { break; // We are guaranteed to have only one active state } } // Now that we are done traversing the current active states, add any new ones. foreach (var newActiveState in newActiveStates) { this.activeStates.Insert(src_hash[i], newActiveState); } } /* (2) Start new activations from the start state(s) */ if (!this.AllowOverlappingInstances && !ended) { continue; } for (int counter = 0; counter < this.numStartStates; counter++) { int startState = this.startStates[counter]; var startStateMap = this.singleEventStateMap[startState]; if (startStateMap != null) { var m = startStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = startStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], this.defaultRegister)) { var newReg = arcinfo.Transfer == null ? this.defaultRegister : arcinfo.Transfer(synctime, batch[i], this.defaultRegister); int ns = arcinfo.toState; while (true) { if (this.isFinal[ns]) { var otherTime = Math.Min(synctime + this.MaxDuration, StreamEvent.InfinitySyncTime); if (!this.IsSyncTimeSimultaneityFree) { var tentativeOutputEntry = this.tentativeOutput.entries[partitionIndex].value; int ind = tentativeOutputEntry.Insert(src_hash[i]); tentativeOutputEntry.Values[ind].other = otherTime; tentativeOutputEntry.Values[ind].key = srckey[i]; tentativeOutputEntry.Values[ind].payload = newReg; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = otherTime; this.batch[this.iter] = newReg; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[ns]) { int index = this.activeStates.Insert(src_hash[i]); this.activeStates.Values[index].key = srckey[i]; this.activeStates.Values[index].state = ns; this.activeStates.Values[index].register = newReg; this.activeStates.Values[index].PatternStartTimestamp = synctime; // Add epsilon arc destinations to stack if (this.epsilonStateMap == null) { break; } if (this.epsilonStateMap[ns] != null) { for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++) { stack.Push(this.epsilonStateMap[ns][cnt2]); } } } if (stack.Count == 0) { break; } ns = stack.Pop(); } if (this.IsDeterministic) { break; // We are guaranteed to have only one successful transition } } } } if (this.IsDeterministic) { break; // We are guaranteed to have only one start state } } } else if (src_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { // Clean active states for stale partitions int seenEventIndex; if (this.activeStates.Count > 0) { var activeVisibleTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .VisibleTraverser(this.activeStates); while (activeVisibleTraverser.Next(out int activeStateIndex, out _)) { var activeState = this.activeStates.Values[activeStateIndex]; if (synctime >= activeState.PatternStartTimestamp + this.MaxDuration) { // Since we know this partition is stale, remove it from seenEvent as well. this.seenEvent.Remove(activeState.key); this.activeStates.Remove(activeStateIndex); } } } // Clean seen events from stale partitions. This enumeration is necessary for stale partitions without active state. seenEventIndex = FastDictionary2 <TPartitionKey, long> .IteratorStart; int partitionIndex; while (this.seenEvent.Iterate(ref seenEventIndex)) { var partitionKey = this.getPartitionKey(this.seenEvent.entries[seenEventIndex].key); if (this.lastSyncTime.Lookup(partitionKey, out partitionIndex) && synctime > this.lastSyncTime.entries[partitionIndex].value) { this.seenEvent.Remove(this.seenEvent.entries[seenEventIndex].key); } } // Clean last synctime and tentative output from stale partitions (these two need to be kept in sync) partitionIndex = FastDictionary2 <TPartitionKey, long> .IteratorStart; while (this.lastSyncTime.Iterate(ref partitionIndex)) { // Check to see if partition is stale if (synctime > this.lastSyncTime.entries[partitionIndex].value) { // Emit tentative output from stale partitions var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput.entries[partitionIndex].value); while (tentativeVisibleTraverser.Next(out int index, out int hash)) { var elem = this.tentativeOutput.entries[partitionIndex].value.Values[index]; this.batch.vsync.col[this.iter] = this.lastSyncTime.entries[partitionIndex].value; this.batch.vother.col[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; this.batch.key.col[this.iter] = elem.key; this.batch.hash.col[this.iter] = hash; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } // Remove the partition var partitionKey = this.lastSyncTime.entries[partitionIndex].key; this.tentativeOutput.Remove(partitionKey); this.lastSyncTime.Remove(partitionKey); } } } // Update dest_* on low watermark in case this event will hit the batch boundary and allocate a new batch OnLowWatermark(synctime); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { var key = srckey[i]; long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { var partitionKey = this.getPartitionKey(key); int partitionIndex = EnsurePartition(partitionKey); if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward { this.seenEvent.Remove(srckey[i]); var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput.entries[partitionIndex].value); if (this.tentativeOutput.Count > 0) { tentativeVisibleTraverser.currIndex = 0; while (tentativeVisibleTraverser.Next(out int index, out int hash)) { var elem = this.tentativeOutput.entries[partitionIndex].value.Values[index]; this.batch.vsync.col[this.iter] = this.lastSyncTime.entries[partitionIndex].value; this.batch.vother.col[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; this.batch.key.col[this.iter] = elem.key; this.batch.hash.col[this.iter] = hash; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } this.tentativeOutput.entries[partitionIndex].value.Clear(); // Clear the tentative output list } this.lastSyncTime.entries[partitionIndex].value = synctime; } } this.batch.vsync.col[this.iter] = synctime; this.batch.vother.col[this.iter] = long.MinValue; this.batch.payload.col[this.iter] = default; this.batch.key.col[this.iter] = key; this.batch.hash.col[this.iter] = src_hash[i]; this.batch.bitvector.col[this.iter >> 6] |= (1L << (this.iter & 0x3f)); this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; fixed(long *bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { if (batch.vsync.col[i] > this.lastSyncTime) { ReachTime(batch.vsync.col[i]); } if (batch.vother.col[i] == StreamEvent.InfinitySyncTime) { } else if (batch.vother.col[i] < batch.vsync.col[i]) { int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = batch.vsync.col[i] + 1; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else { int index = this.intervalMap.Insert(batch.hash.col[i]); this.intervalMap.Values[index].Populate(batch.key.col[i], batch[i], batch.hash.col[i]); this.endPointHeap.Insert(batch.vother.col[i], index); } } else if (batch.vother.col[i] == StreamEvent.PunctuationOtherTime) { ReachTime(batch.vsync.col[i]); int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = batch.vother.col[i]; this.output.key.col[ind] = batch.key.col[i]; this.output.payload.col[ind] = default; this.output.hash.col[ind] = batch.hash.col[i]; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; fixed(long *bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { if (batch.vsync.col[i] > this.lastSyncTime) { ReachTime(batch.vsync.col[i]); } if (batch.vother.col[i] == StreamEvent.InfinitySyncTime) // For start events, copy directly across { int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = StreamEvent.InfinitySyncTime; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else if (batch.vother.col[i] > batch.vsync.col[i]) // For intervals, just extend the duration { int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = batch.vother.col[i] + this.duration; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else { int index = this.endPointMap.Insert(batch.hash.col[i]); this.endPointMap.Values[index].Populate(batch.key.col[i], batch[i], batch.vother.col[i], batch.hash.col[i]); this.endPointHeap.Insert(batch.vsync.col[i] + this.duration, index); } } else if (batch.vother.col[i] == long.MinValue) { ReachTime(batch.vsync.col[i]); int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = long.MinValue; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = default; this.output.hash.col[ind] = batch.hash.col[i]; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; fixed(long *bv = batch.bitvector.col) fixed(long *vsync = batch.vsync.col) fixed(long *vother = batch.vother.col) fixed(int *hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { if (vsync[i] > vother[i]) // We have an end edge { ReachTime(-1, vsync[i]); } else { // Check to see if the key is already being tracked if (!this.lastDataTimeDictionary.Lookup(batch.key.col[i], out int keyIndex)) { keyIndex = AllocatePartition(batch.key.col[i]); } ReachTime(keyIndex, vsync[i]); // Check to see if advancing time removed the key if (!this.lastDataTimeDictionary.Lookup(batch.key.col[i], out keyIndex)) { keyIndex = AllocatePartition(batch.key.col[i]); } if (!this.stateDictionary.entries[keyIndex].value.Any()) { this.orderedKeys.AddLast(new LinkedListNode <TKey>(batch.key.col[i])); } else { var oldThreshold = Math.Min(this.lastDataTimeDictionary.entries[keyIndex].value + this.sessionTimeout, this.windowEndTimeDictionary.entries[keyIndex].value); var newThreshold = Math.Min(vsync[i] + this.sessionTimeout, this.windowEndTimeDictionary.entries[keyIndex].value); if (newThreshold > oldThreshold) { var node = this.orderedKeys.Find(batch.key.col[i]); this.orderedKeys.Remove(node); this.orderedKeys.AddLast(node); } } this.lastDataTimeDictionary.entries[keyIndex].value = vsync[i]; this.stateDictionary.entries[keyIndex].value.Enqueue(new ActiveEvent { Key = batch.key.col[i], Sync = vsync[i], Hash = hash[i], Payload = batch.payload.col[i], }); int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i]; this.output.vother.col[ind] = StreamEvent.InfinitySyncTime; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch.payload.col[i]; this.output.hash.col[ind] = hash[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } else if (vother[i] == long.MinValue) { ReachTime(-1, vsync[i]); int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i]; this.output.vother.col[ind] = long.MinValue; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch.payload.col[i]; this.output.hash.col[ind] = hash[i]; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var stack = new Stack <int>(); var activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates); var tentativeFindTraverser = new FastMap <OutputEvent <TKey, TRegister> > .FindTraverser(this.tentativeOutput); var tentativeVisibleTraverser = new FastMap <OutputEvent <TKey, TRegister> > .VisibleTraverser(this.tentativeOutput); var count = batch.Count; var dest_vsync = this.batch.vsync.col; var dest_vother = this.batch.vother.col; var destkey = this.batch.key.col; var dest_hash = this.batch.hash.col; var srckey = batch.key.col; fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col) { fixed(int *src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0) { long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { int index; if (synctime > this.lastSyncTime) // move time forward { this.seenEvent.Clear(); if (this.tentativeOutput.Count > 0) { tentativeVisibleTraverser.currIndex = 0; while (tentativeVisibleTraverser.Next(out index, out int hash)) { var elem = this.tentativeOutput.Values[index]; dest_vsync[this.iter] = this.lastSyncTime; dest_vother[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; destkey[this.iter] = elem.key; dest_hash[this.iter] = hash; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } this.tentativeOutput.Clear(); // Clear the tentative output list } this.lastSyncTime = synctime; } if (this.seenEvent.Lookup(srckey[i], out index)) // Incoming event is a simultaneous one { if (this.seenEvent.entries[index].value == 1) // Detecting first duplicate, need to adjust state { this.seenEvent.entries[index].value = 2; // Delete tentative output for that key if (tentativeFindTraverser.Find(src_hash[i])) { while (tentativeFindTraverser.Next(out index)) { if (this.keyEqualityComparer(this.tentativeOutput.Values[index].key, srckey[i])) { tentativeFindTraverser.Remove(); } } } // Delete active states for that key if (activeFindTraverser.Find(src_hash[i])) { while (activeFindTraverser.Next(out index)) { if (this.keyEqualityComparer(this.activeStates.Values[index].key, srckey[i])) { activeFindTraverser.Remove(); } } } } // Dont process this event continue; } else { this.seenEvent.Insert(ref index, srckey[i], 1); } } /* (1) Process currently active states */ bool ended = true; if (activeFindTraverser.Find(src_hash[i])) { int orig_index; while (activeFindTraverser.Next(out int index)) { orig_index = index; var state = this.activeStates.Values[index]; if (!this.keyEqualityComparer(state.key, srckey[i])) { continue; } if (state.PatternStartTimestamp + this.MaxDuration > synctime) { var currentStateMap = this.singleEventStateMap[state.state]; if (currentStateMap != null) { var m = currentStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = currentStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], state.register)) { var newReg = arcinfo.Transfer == null ? state.register : arcinfo.Transfer(synctime, batch[i], state.register); int ns = arcinfo.toState; while (true) { if (this.isFinal[ns]) { if (!this.IsSyncTimeSimultaneityFree) { int ind = this.tentativeOutput.Insert(src_hash[i]); this.tentativeOutput.Values[ind].other = state.PatternStartTimestamp + this.MaxDuration; this.tentativeOutput.Values[ind].key = srckey[i]; this.tentativeOutput.Values[ind].payload = newReg; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = state.PatternStartTimestamp + this.MaxDuration; this.batch[this.iter] = newReg; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[ns]) { if (index == -1) { index = this.activeStates.Insert(src_hash[i]); } this.activeStates.Values[index].key = srckey[i]; this.activeStates.Values[index].state = ns; this.activeStates.Values[index].register = newReg; this.activeStates.Values[index].PatternStartTimestamp = state.PatternStartTimestamp; index = -1; ended = false; // Add epsilon arc destinations to stack if (this.epsilonStateMap == null) { break; } if (this.epsilonStateMap[ns] != null) { for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++) { stack.Push(this.epsilonStateMap[ns][cnt2]); } } } if (stack.Count == 0) { break; } ns = stack.Pop(); } if (this.IsDeterministic) { break; // We are guaranteed to have only one successful transition } } } } } if (index == orig_index) { activeFindTraverser.Remove(); } if (this.IsDeterministic) { break; // We are guaranteed to have only one active state } } } /* (2) Start new activations from the start state(s) */ if (!this.AllowOverlappingInstances && !ended) { continue; } for (int counter = 0; counter < this.numStartStates; counter++) { int startState = this.startStates[counter]; var startStateMap = this.singleEventStateMap[startState]; if (startStateMap != null) { var m = startStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = startStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], this.defaultRegister)) { var newReg = arcinfo.Transfer == null ? this.defaultRegister : arcinfo.Transfer(synctime, batch[i], this.defaultRegister); int ns = arcinfo.toState; while (true) { if (this.isFinal[ns]) { if (!this.IsSyncTimeSimultaneityFree) { int ind = this.tentativeOutput.Insert(src_hash[i]); this.tentativeOutput.Values[ind].other = synctime + this.MaxDuration; this.tentativeOutput.Values[ind].key = srckey[i]; this.tentativeOutput.Values[ind].payload = newReg; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = synctime + this.MaxDuration; this.batch[this.iter] = newReg; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[ns]) { int index = this.activeStates.Insert(src_hash[i]); this.activeStates.Values[index].key = srckey[i]; this.activeStates.Values[index].state = ns; this.activeStates.Values[index].register = newReg; this.activeStates.Values[index].PatternStartTimestamp = synctime; // Add epsilon arc destinations to stack if (this.epsilonStateMap == null) { break; } if (this.epsilonStateMap[ns] != null) { for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++) { stack.Push(this.epsilonStateMap[ns][cnt2]); } } } if (stack.Count == 0) { break; } ns = stack.Pop(); } if (this.IsDeterministic) { break; // We are guaranteed to have only one successful transition } } } } if (this.IsDeterministic) { break; // We are guaranteed to have only one start state } } } else if (batch.vother.col[i] < 0 && !this.IsSyncTimeSimultaneityFree) { long synctime = src_vsync[i]; if (synctime > this.lastSyncTime) // move time forward { this.seenEvent.Clear(); if (this.tentativeOutput.Count > 0) { tentativeVisibleTraverser.currIndex = 0; while (tentativeVisibleTraverser.Next(out int index, out int hash)) { var elem = this.tentativeOutput.Values[index]; this.batch.vsync.col[this.iter] = this.lastSyncTime; this.batch.vother.col[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; this.batch.key.col[this.iter] = elem.key; this.batch.hash.col[this.iter] = hash; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); } } this.tentativeOutput.Clear(); // Clear the tentative output list } this.lastSyncTime = synctime; } OnPunctuation(synctime); } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { this.batchIter = batch.iter; TPayload[] sourcePayload = batch.payload.col; TKey[] sourceKey = batch.key.col; fixed(int *sourceHash = batch.hash.col) fixed(long *sourceBitVector = batch.bitvector.col) fixed(long *sourceVSync = batch.vsync.col) fixed(long *sourceVOther = batch.vother.col) { int count = batch.Count; int * sourceHashPtr = sourceHash; long *sourceVSyncPtr = sourceVSync; long *sourceVOtherPtr = sourceVOther; for (int row = 0; row < count; row++) { if ((sourceBitVector[row >> 6] & (1L << (row & 0x3f))) == 0 || *sourceVOtherPtr == long.MinValue) { long startTime = *sourceVSyncPtr; long endTime = *sourceVOtherPtr; int hash = *sourceHashPtr; AdvanceTime(startTime); bool isPunctuation = endTime == long.MinValue; bool isInsert = startTime < endTime; bool isStartEdge = isInsert && endTime == StreamEvent.InfinitySyncTime; bool isEndEdge = !isInsert; if (isPunctuation) { AddToBatch(startTime, long.MinValue, ref sourceKey[row], ref sourcePayload[row], hash); } else if (isStartEdge) { // Add starting edge { vSync = startTime, vOther = StreamEvent.InfinitySyncTime }. AddToBatch( startTime, StreamEvent.InfinitySyncTime, ref sourceKey[row], ref sourcePayload[row], hash); // Add to active edges list to handle repeat at beats (and waiting for closing edge). int index = this.edges.Insert(hash); this.edges.Values[index].Populate( startTime, ref sourceKey[row], ref sourcePayload[row]); } else if (isEndEdge) { bool notCurrentlyOnBeat = startTime != this.currBeatTime; long edgeStartTime = endTime; long edgeEndTime = startTime; if (notCurrentlyOnBeat) { // Edges are only open if not on a beat. long lastBeatTime = this.currBeatTime - this.period; bool edgeStartedBeforeLastBeat = edgeStartTime < lastBeatTime; if (edgeStartedBeforeLastBeat) { // Add closing edge { vSync = edgeEndTime, vOther = lastBeatTime }. AddToBatch( edgeEndTime, lastBeatTime, ref sourceKey[row], ref sourcePayload[row], hash); } else { // Add closing edge { vSync = edgeEndTime, vOther = edgeStartTime }. AddToBatch( edgeEndTime, edgeStartTime, ref sourceKey[row], ref sourcePayload[row], hash); } } // Remove from active edges list. var edgesTraversal = this.edges.Find(hash); while (edgesTraversal.Next(out int index)) { var temp = this.edges.Values[index]; if (AreSame(edgeStartTime, ref sourceKey[row], ref sourcePayload[row], ref temp)) { edgesTraversal.Remove(); break; } } } else { long nextBeatTime = startTime == this.currBeatTime ? this.currBeatTime + this.period : this.currBeatTime; bool isLastBeatForInterval = endTime <= nextBeatTime; if (isLastBeatForInterval) { // Add interval { vSync = startTime, vOther = endTime }. AddToBatch(startTime, endTime, ref sourceKey[row], ref sourcePayload[row], hash); // No need to add to active list as interval ends <= nextBeatTime. } else { // Add interval { vSync = startTime, vOther = nextBeatTime }. AddToBatch(startTime, nextBeatTime, ref sourceKey[row], ref sourcePayload[row], hash); // Add to active list to handle repeat at beats. int index = this.intervals.Insert(hash); this.intervals.Values[index].Populate(endTime, ref sourceKey[row], ref sourcePayload[row]); } } } // Advance pointers. sourceHashPtr++; sourceVSyncPtr++; sourceVOtherPtr++; } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; fixed(long *bv = batch.bitvector.col) fixed(long *vsync = batch.vsync.col) fixed(long *vother = batch.vother.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { if (vsync[i] > this.lastSyncTime) { ReachTime(vsync[i]); } if (vother[i] == StreamEvent.InfinitySyncTime) // Start edge { int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i] - ((vsync[i] - this.offset) % this.progress + this.progress) % this.progress; this.output.vother.col[ind] = StreamEvent.InfinitySyncTime; this.output.key.col[ind] = batch.key.col[i]; this.output.payload.col[ind] = batch.payload.col[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else if (vother[i] > vsync[i]) // Interval { int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i] - ((vsync[i] - this.offset) % this.progress + this.progress) % this.progress; var temp = Math.Max(vother[i] + this.skip - 1, vsync[i] + this.width); this.output.vother.col[ind] = temp - ((temp - (this.offset + this.width)) % this.skip + this.skip) % this.skip; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else // End edge { var temp = Math.Max(vsync[i] + this.skip - 1, vother[i] + this.width); int index = this.intervalMap.Insert(batch.hash.col[i]); this.intervalMap.Values[index].Populate(batch.key.col[i], batch[i], batch.hash.col[i], vother[i] - ((vother[i] - this.offset) % this.progress + this.progress) % this.progress); this.endPointHeap.Insert(temp - ((temp - (this.offset + this.width)) % this.skip + this.skip) % this.skip, index); } } else if (vother[i] == long.MinValue) // Punctuation { if (vsync[i] > this.lastSyncTime) { ReachTime(vsync[i]); } int ind = this.output.Count++; this.output.vsync.col[ind] = vsync[i] - ((vsync[i] - this.offset) % this.progress + this.progress) % this.progress; this.output.vother.col[ind] = long.MinValue; this.output.key.col[ind] = batch.key.col[i]; this.output.payload.col[ind] = default; this.output.hash.col[ind] = batch.hash.col[i]; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } batch.Free(); }
public void OnNext(StreamMessage <TKey, TSpray> batch) { if (this.multicast) { for (int i = 0; i < this.totalBranches; i++) { this.pool.Get(out StreamMessage <TKey, TSpray> _outbatch); _outbatch.CloneFrom(batch, false); this.Observers[i].OnNext(_outbatch); } batch.Free(); return; } // If this batch contains any punctuation, we need to broadcast them to all observers. StreamMessage <TKey, TSpray> broadcastMaster = null; for (int i = batch.Count - 1; i >= 0; i--) { if (batch.vother.col[i] < 0) { // Create a master broadcast batch that we can clone from. // TODO: maybe it's better to allocate a new batch without deleted data gaps? this.pool.Get(out broadcastMaster); broadcastMaster.CloneFrom(batch); broadcastMaster.bitvector = broadcastMaster.bitvector.MakeWritable(this.pool.bitvectorPool); // Since we only care about punctuations, delete everything for (int deletingIndex = 0; deletingIndex <= broadcastMaster.Count >> 6; deletingIndex++) { broadcastMaster.bitvector.col[deletingIndex] = ~(0L); } break; } } int originalBatchRecipient; if (this.spraySortOrderComparer == null) { originalBatchRecipient = this.l1_spray; this.Observers[this.l1_spray].OnNext(batch); this.l1_spray++; if (this.l1_spray == this.totalBranches) { this.l1_spray = 0; } } else { if (this.first || (this.spraySortOrderComparerFunc(this.lastElem, batch[0]) == 0)) { this.first = false; this.lastElem = batch[batch.Count - 1]; originalBatchRecipient = this.l1_spray; this.Observers[this.l1_spray].OnNext(batch); } else { this.lastElem = batch[batch.Count - 1]; this.l1_spray++; if (this.l1_spray == this.totalBranches) { this.l1_spray = 0; } originalBatchRecipient = this.l1_spray; this.Observers[this.l1_spray].OnNext(batch); } } if (broadcastMaster != null) { // Broadcast to all except the observer that received the current batch int lastBroadcastIndex = this.totalBranches - 1; if (lastBroadcastIndex == originalBatchRecipient) { lastBroadcastIndex--; } for (int i = 0; i < this.totalBranches; i++) { if (i == originalBatchRecipient) { continue; // skip observer that received the current batch } if (i == lastBroadcastIndex) { this.Observers[i].OnNext(broadcastMaster); } else { this.pool.Get(out StreamMessage <TKey, TSpray> broadcastClone); broadcastClone.CloneFrom(broadcastMaster); this.Observers[i].OnNext(broadcastClone); } } } }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; batch.bitvector = batch.bitvector.MakeWritable(this.pool.bitvectorPool); fixed(long *bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0 || batch.vother.col[i] < 0) { if (batch.vother.col[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { int index = FastDictionary <TPartitionKey, PartitionEntry> .IteratorStart; while (this.partitionData.Iterate(ref index)) { var p = this.partitionData.entries[index].value; if (batch.vsync.col[i] == StreamEvent.InfinitySyncTime) { OutputAllEvents(p); } else { OutputCompletedIntervals(p); } p.lastCti = Math.Max(batch.vsync.col[i], p.lastCti); p.lastSyncTime = Math.Max(batch.vsync.col[i], p.lastSyncTime); } AddLowWatermarkToBatch(batch.vother.col[i]); continue; } var partitionKey = this.getPartitionKey(batch.key.col[i]); PartitionEntry partition; if (!this.partitionData.Lookup(partitionKey, out int tempIndex)) { this.partitionData.Insert(ref tempIndex, partitionKey, partition = new PartitionEntry()); } else { partition = this.partitionData.entries[tempIndex].value; } if (batch.vother.col[i] == PartitionedStreamEvent.PunctuationOtherTime) { partition.lastCti = batch.vsync.col[i]; partition.lastSyncTime = batch.vsync.col[i]; } else if (batch.vsync.col[i] < batch.vother.col[i]) // Start edge or interval { var evt = new ActiveEvent { End = batch.vother.col[i], Payload = batch[i], Key = batch.key.col[i], Hash = batch.hash.col[i] }; if (!partition.eventMap.TryGetValue(batch.vsync.col[i], out FastDictionary2 <ActiveEvent, int> entry)) { this.dictPool.Get(out entry); partition.eventMap.Add(batch.vsync.col[i], entry); } if (!entry.Lookup(evt, out int index)) { entry.Insert(evt, 1); } else { entry.entries[index].value++; } } else // end edge { // lookup corresponding start edge var lookupevt = new ActiveEvent { End = StreamEvent.InfinitySyncTime, Payload = batch[i], Key = batch.key.col[i], Hash = batch.hash.col[i] }; if (!partition.eventMap.TryGetValue(batch.vother.col[i], out FastDictionary2 <ActiveEvent, int> entry)) { throw new InvalidOperationException("Found end edge without corresponding start edge"); } if (!entry.Lookup(lookupevt, out int index)) { throw new InvalidOperationException("Found end edge without corresponding start edge"); } // Set interval payload to the payload of the original start-edge // (in case they are different due to an optimized payload equality comparer) lookupevt.Payload = entry.entries[index].key.Payload; // delete the start edge entry.entries[index].value--; if (entry.entries[index].value == 0) { entry.Remove(lookupevt); } // insert interval lookupevt.End = batch.vsync.col[i]; if (!entry.Lookup(lookupevt, out index)) { entry.Insert(lookupevt, 1); } else { entry.entries[index].value++; } OutputCompletedIntervals(partition); // Can make this more efficient by trying only if the first event in index got completed } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; var srckey = batch.key.col; SavedEventList <TKey, TPayload> sevref = default; fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col) { fixed(int *src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0 || batch.vother.col[i] < 0) { long synctime = src_vsync[i]; int index; if (synctime > this.lastSyncTime) // move time forward { ProcessCurrentTimestamp(); this.lastSyncTime = synctime; } if (batch.vother.col[i] < 0) { OnPunctuation(synctime); continue; } bool done = false; if (this.eventListTraverser.Find(src_hash[i])) { while (this.eventListTraverser.Next(out index)) { var state = this.currentTimestampEventList.Values[index]; if (this.keyEqualityComparer(state.key, srckey[i])) { state.payloads.Add(batch.payload.col[i]); done = true; break; } } } if (!done) { index = this.currentTimestampEventList.Insert(src_hash[i]); sevref.payloads = new List <TPayload>(10) { batch.payload.col[i] }; sevref.key = srckey[i]; this.currentTimestampEventList.Values[index] = sevref; } } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { var count = batch.Count; fixed(long *bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { var partition = this.getPartitionKey(batch.key.col[i]); if (!this.lastSyncTimeDictionary.Lookup(partition, out int timeIndex)) { AllocatePartition(partition, batch.vsync.col[i]); } else { ReachTime(partition, batch.vsync.col[i]); } if (batch.vother.col[i] == StreamEvent.InfinitySyncTime) { // For start events, we copy directly to the output batch // and add them to the list of events that may need to be clipped int ind = this.output.Count++; var sync = batch.vsync.col[i]; this.output.vsync.col[ind] = sync; this.output.vother.col[ind] = StreamEvent.InfinitySyncTime; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } this.syncTimeMapDictionary.Lookup(partition, out int mapIndex); var syncTimeMap = this.syncTimeMapDictionary.entries[mapIndex].value; if (!syncTimeMap.TryGetValue(sync, out MultiSet <ActiveEvent> multiSet)) { multiSet = new MultiSet <ActiveEvent>(); syncTimeMap.Add(sync, multiSet); } multiSet.Add(new ActiveEvent { Hash = batch.hash.col[i], Key = batch.key.col[i], Payload = batch[i] }); } else if (batch.vother.col[i] > batch.vsync.col[i]) { // For intervals, we clip the limit and copy to the output batch int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = Math.Min(batch.vother.col[i], batch.vsync.col[i] + this.limit); this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = batch[i]; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else { var sync = batch.vsync.col[i]; var other = batch.vother.col[i]; // For end edges, if the delta is greater than the limit, then ignore, // otherwise copy directly over if (other + this.limit < sync) { continue; } var payload = batch[i]; int ind = this.output.Count++; this.output.vsync.col[ind] = sync; this.output.vother.col[ind] = other; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = payload; this.output.hash.col[ind] = batch.hash.col[i]; if (this.output.Count == Config.DataBatchSize) { FlushContents(); } // Remove the corresponding start edge from the waiting list this.syncTimeMapDictionary.Lookup(partition, out int mapIndex); var syncTimeMap = this.syncTimeMapDictionary.entries[mapIndex].value; syncTimeMap[other].Remove(new ActiveEvent { Payload = payload, Key = batch.key.col[i], Hash = batch.hash.col[i] }); if (syncTimeMap[other].IsEmpty) { syncTimeMap.Remove(other); } } } else if (batch.vother.col[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { ReachTime(batch.vsync.col[i]); int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = long.MinValue; this.output.key.col[ind] = default; this.output[ind] = default; this.output.hash.col[ind] = 0; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } else if (batch.vother.col[i] == PartitionedStreamEvent.PunctuationOtherTime) { var partition = this.getPartitionKey(batch.key.col[i]); ReachTime(partition, batch.vsync.col[i]); int ind = this.output.Count++; this.output.vsync.col[ind] = batch.vsync.col[i]; this.output.vother.col[ind] = long.MinValue; this.output.key.col[ind] = batch.key.col[i]; this.output[ind] = default; this.output.hash.col[ind] = batch.hash.col[i]; this.output.bitvector.col[ind >> 6] |= (1L << (ind & 0x3f)); if (this.output.Count == Config.DataBatchSize) { FlushContents(); } } } } batch.Free(); }
public override unsafe void OnNext(StreamMessage <Empty, TPayload> batch) { var stack = new Stack <int>(); var activeFindTraverser = new FastLinkedList <GroupedActiveState <Empty, TRegister> > .ListTraverser(this.activeStates); var tentativeFindTraverser = new FastLinkedList <OutputEvent <Empty, TRegister> > .ListTraverser(this.tentativeOutput); var tentativeOutputIndex = 0; var count = batch.Count; var dest_vsync = this.batch.vsync.col; var dest_vother = this.batch.vother.col; var destkey = this.batch.key.col; var dest_hash = this.batch.hash.col; var srckey = batch.key.col; fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col) { fixed(int *src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0) { long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { if (synctime > this.lastSyncTime) // move time forward { this.seenEvent = 0; if (this.tentativeOutput.Count > 0) { tentativeOutputIndex = 0; while (this.tentativeOutput.Iterate(ref tentativeOutputIndex)) { var elem = this.tentativeOutput.Values[tentativeOutputIndex]; dest_vsync[this.iter] = this.lastSyncTime; dest_vother[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; dest_hash[this.iter] = 0; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } this.tentativeOutput.Clear(); // Clear the tentative output list } this.lastSyncTime = synctime; } if (this.seenEvent > 0) // Incoming event is a simultaneous one { if (this.seenEvent == 1) // Detecting first duplicate, need to adjust state { this.seenEvent = 2; // Delete tentative output for that key this.tentativeOutput.Clear(); // Delete active states for that key this.activeStates.Clear(); } // Dont process this event continue; } else { this.seenEvent = 1; } } /* (1) Process currently active states */ if (activeFindTraverser.Find()) { int orig_index; while (activeFindTraverser.Next(out int index)) { orig_index = index; var state = this.activeStates.Values[index]; if (state.PatternStartTimestamp + this.MaxDuration > synctime) { var currentStateMap = this.singleEventStateMap[state.state]; if (currentStateMap != null) { var m = currentStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = currentStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], state.register)) { var newReg = arcinfo.Transfer == null ? state.register : arcinfo.Transfer(synctime, batch[i], state.register); int ns = arcinfo.toState; while (true) { if (this.isFinal[ns]) { var otherTime = Math.Min(state.PatternStartTimestamp + this.MaxDuration, StreamEvent.InfinitySyncTime); if (!this.IsSyncTimeSimultaneityFree) { int ind = this.tentativeOutput.Insert(); this.tentativeOutput.Values[ind].other = otherTime; this.tentativeOutput.Values[ind].key = srckey[i]; this.tentativeOutput.Values[ind].payload = newReg; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = otherTime; this.batch[this.iter] = newReg; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[ns]) { if (index == -1) { index = this.activeStates.Insert(); } this.activeStates.Values[index].key = srckey[i]; this.activeStates.Values[index].state = ns; this.activeStates.Values[index].register = newReg; this.activeStates.Values[index].PatternStartTimestamp = state.PatternStartTimestamp; index = -1; // Add epsilon arc destinations to stack if (this.epsilonStateMap == null) { break; } if (this.epsilonStateMap[ns] != null) { for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++) { stack.Push(this.epsilonStateMap[ns][cnt2]); } } } if (stack.Count == 0) { break; } ns = stack.Pop(); } } } } } if (index == orig_index) { activeFindTraverser.Remove(); } } } /* (2) Start new activations from the start state(s) */ if (!this.AllowOverlappingInstances && this.activeStates.Count > 0) { continue; } for (int counter = 0; counter < this.numStartStates; counter++) { int startState = this.startStates[counter]; var startStateMap = this.singleEventStateMap[startState]; if (startStateMap != null) { var m = startStateMap.Length; for (int cnt = 0; cnt < m; cnt++) { var arcinfo = startStateMap[cnt]; if (arcinfo.Fence(synctime, batch[i], this.defaultRegister)) { var newReg = arcinfo.Transfer == null ? this.defaultRegister : arcinfo.Transfer(synctime, batch[i], this.defaultRegister); int ns = arcinfo.toState; while (true) { if (this.isFinal[ns]) { var otherTime = Math.Min(synctime + this.MaxDuration, StreamEvent.InfinitySyncTime); if (!this.IsSyncTimeSimultaneityFree) { int ind = this.tentativeOutput.Insert(); this.tentativeOutput.Values[ind].other = otherTime; this.tentativeOutput.Values[ind].key = srckey[i]; this.tentativeOutput.Values[ind].payload = newReg; } else { dest_vsync[this.iter] = synctime; dest_vother[this.iter] = otherTime; this.batch[this.iter] = newReg; destkey[this.iter] = srckey[i]; dest_hash[this.iter] = src_hash[i]; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } if (this.hasOutgoingArcs[ns]) { int index = this.activeStates.Insert(); this.activeStates.Values[index].key = srckey[i]; this.activeStates.Values[index].state = ns; this.activeStates.Values[index].register = newReg; this.activeStates.Values[index].PatternStartTimestamp = synctime; // Add epsilon arc destinations to stack if (this.epsilonStateMap == null) { break; } if (this.epsilonStateMap[ns] != null) { for (int cnt2 = 0; cnt2 < this.epsilonStateMap[ns].Length; cnt2++) { stack.Push(this.epsilonStateMap[ns][cnt2]); } } } if (stack.Count == 0) { break; } ns = stack.Pop(); } } } } } } else if (batch.vother.col[i] < 0) { long synctime = src_vsync[i]; if (!this.IsSyncTimeSimultaneityFree) { if (synctime > this.lastSyncTime) // move time forward { this.seenEvent = 0; if (this.tentativeOutput.Count > 0) { tentativeOutputIndex = 0; while (this.tentativeOutput.Iterate(ref tentativeOutputIndex)) { var elem = this.tentativeOutput.Values[tentativeOutputIndex]; this.batch.vsync.col[this.iter] = this.lastSyncTime; this.batch.vother.col[this.iter] = elem.other; this.batch.payload.col[this.iter] = elem.payload; this.batch.hash.col[this.iter] = 0; this.iter++; if (this.iter == Config.DataBatchSize) { FlushContents(); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } this.tentativeOutput.Clear(); // Clear the tentative output list } this.lastSyncTime = synctime; } } // Update dest_* on punctuation in case this event will hit the batch boundary and allocate a new batch OnPunctuation(synctime); dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } } } } batch.Free(); }