private void CheckAndRelease(StreamMessage <Empty, TPayload> batch) { if (batch != null) { batch.Release(); batch.Return(); } }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { OnLowWatermark(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = PartitionedStreamEvent.LowWatermarkOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else if (col_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation var p = this.getPartitionKey(colkey[i]); PartitionEntry partitionEntry; if (!this.partitionData.Lookup(p, out int partitionIndex)) { this.partitionData.Insert(p, partitionEntry = new PartitionEntry { lastSyncTime = col_vsync[i], ecq = new CircularBuffer <EcqState>(this.hopsPerDuration) }); } else { partitionEntry = this.partitionData.entries[partitionIndex].value; } OnPunctuation(partitionEntry, col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; var partition = this.getPartitionKey(colkey[i]); HeldState <TState> heldState; PartitionEntry entry; // Handle time moving forward if (!this.partitionData.Lookup(partition, out int pIndex)) { this.partitionData.Insert(partition, entry = new PartitionEntry { lastSyncTime = syncTime, ecq = new CircularBuffer <EcqState>(this.hopsPerDuration) }); } else if (syncTime > (entry = this.partitionData.entries[pIndex].value).lastSyncTime) { AdvanceTime(entry, syncTime); } // Need to retrieve the key from the dictionary if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; entry.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else if (entry.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { if (heldState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { this.batch.iter = batch.iter; FlushContents(); this.batch.iter = batch.iter; } } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } if (col_vsync[i] < col_vother[i]) // insert event { heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active++; // Update ECQ if (col_vother[i] < StreamEvent.InfinitySyncTime) { EcqState state; StateAndActive <TState> newState; int index; if (entry.ecq.Count > 0) { state = entry.ecq.PeekLast(); if (state.timestamp != col_vother[i]) { state = new EcqState { timestamp = col_vother[i] }; this.ecqEntryPool.Get(out state.states); newState = new StateAndActive <TState> { state = this.initialState() }; state.states.Lookup(colkey[i], col_hash[i], out index); state.states.Insert(ref index, colkey[i], newState); entry.ecq.Enqueue(ref state); } else { if (!state.states.Lookup(colkey[i], col_hash[i], out index)) { newState = new StateAndActive <TState> { state = this.initialState() }; state.states.Insert(ref index, colkey[i], newState); } else { newState = state.states.entries[index].value; } } } else { state = new EcqState { timestamp = col_vother[i] }; this.ecqEntryPool.Get(out state.states); newState = new StateAndActive <TState> { state = this.initialState() }; state.states.Lookup(colkey[i], col_hash[i], out index); state.states.Insert(ref index, colkey[i], newState); entry.ecq.Enqueue(ref state); } newState.state = this.accumulate(newState.state, col_vsync[i], colpayload[i]); newState.active++; } } else // is a retraction { heldState.state = this.deaccumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active--; } } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { OnLowWatermark(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = PartitionedStreamEvent.LowWatermarkOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else if (col_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation var p = this.getPartitionKey(colkey[i]); PartitionEntry partitionEntry; if (!this.partitionData.Lookup(p, out int partitionIndex)) { this.partitionData.Insert(p, partitionEntry = new PartitionEntry { lastSyncTime = col_vsync[i] }); } else { partitionEntry = this.partitionData.entries[partitionIndex].value; } OnPunctuation(partitionEntry, col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; var partition = this.getPartitionKey(colkey[i]); HeldState <TState> heldState; PartitionEntry entry; // Handle time moving forward if (!this.partitionData.Lookup(partition, out int pIndex)) { this.partitionData.Insert(partition, entry = new PartitionEntry { lastSyncTime = syncTime }); } else if (syncTime > (entry = this.partitionData.entries[pIndex].value).lastSyncTime) { /* Issue start edges for held aggregates */ foreach (int iter1 in entry.heldAggregates) { var iter1entry = this.aggregateByKey.entries[iter1]; if (partition.Equals(this.getPartitionKey(iter1entry.key))) { int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.computeResult(iter1entry.value.state); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } } // Time has moved forward, clear the held aggregates entry.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime entry.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; entry.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else if (entry.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch) { var count = batch.Count; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { if (this.currentState != null && this.held) { int c = this.batch.Count; this.batch.vsync.col[c] = this.currentState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.computeResult(this.currentState.state); this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } this.held = false; } // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } if (this.currentState == null) { this.currentState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.held = true; } else { if (syncTime > this.currentState.timestamp) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.computeResult(this.currentState.state); this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } this.currentState.timestamp = syncTime; this.held = true; } } // It's always a start edge this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]); } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { AdvanceTime(syncTime); } if (this.currentState == null) { this.currentEcqHeldState = null; this.currentState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.held = true; // No output because initial state is empty } else { if (syncTime > this.currentState.timestamp) { if (this.currentState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.computeResult(this.currentState.state); this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.currentState.timestamp = syncTime; this.held = true; } } if (col_vsync[i] < col_vother[i]) // insert event { this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active++; // Update ECQ if (col_vother[i] < StreamEvent.InfinitySyncTime) { if ((this.currentEcqHeldState == null) || (this.currentEcqHeldState.timestamp != col_vother[i])) { if (this.ecq.Count > 0) { this.currentEcqHeldState = this.ecq.PeekLast(); if (this.currentEcqHeldState.timestamp != col_vother[i]) { this.currentEcqHeldState = new HeldState <TState> { state = this.initialState(), timestamp = col_vother[i] }; this.ecq.Enqueue(ref this.currentEcqHeldState); } } else { this.currentEcqHeldState = new HeldState <TState> { state = this.initialState(), timestamp = col_vother[i] }; this.ecq.Enqueue(ref this.currentEcqHeldState); } } this.currentEcqHeldState.state = this.accumulate(this.currentEcqHeldState.state, col_vsync[i], colpayload[i]); this.currentEcqHeldState.active++; } } else // is a retraction { this.currentState.state = this.deaccumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active--; } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { int iter1 = FastDictionary <TKey, TState> .IteratorStart; while (this.heldAggregates.Iterate(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = this.lastSyncTime; this.batch.vother.col[c] = this.lastSyncTime + this.hop; this.batch.payload.col[c] = this.computeResult(iter1entry.value); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = iter1entry.key.GetHashCode(); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } if (hasDisposableState) { DisposeStateLocal(); } this.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!this.heldAggregates.Lookup(colkey[i], col_hash[i], out int index)) { this.heldAggregates.Insert(ref index, colkey[i], this.initialState()); } var entries = this.heldAggregates.entries; entries[index].value = this.accumulate(entries[index].value, col_vsync[i], colpayload[i]); } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { AdvanceTime(syncTime); } // Need to retrieve the key from the dictionary HeldState <TState> heldState; if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else if (this.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { if (heldState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { this.batch.iter = batch.iter; FlushContents(); this.batch.iter = batch.iter; } } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } if (col_vsync[i] < col_vother[i]) // insert event { heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active++; // Update ECQ if (col_vother[i] < StreamEvent.InfinitySyncTime) { FastDictionary <TKey, StateAndActive <TState> > state; int index; if (this.ecq.Count > 0) { if (!this.ecq.TryGetValue(col_vother[i], out state)) { this.ecqEntryPool.Get(out state); state.Lookup(colkey[i], col_hash[i], out index); state.Insert(ref index, colkey[i], new StateAndActive <TState> { state = this.initialState() }); this.ecq.Add(col_vother[i], state); } else { if (!state.Lookup(colkey[i], col_hash[i], out index)) { state.Insert(ref index, colkey[i], new StateAndActive <TState> { state = this.initialState() }); } } } else { this.ecqEntryPool.Get(out state); state.Lookup(colkey[i], col_hash[i], out index); state.Insert(ref index, colkey[i], new StateAndActive <TState> { state = this.initialState() }); this.ecq.Add(col_vother[i], state); } state.entries[index].value.state = this.accumulate(state.entries[index].value.state, col_vsync[i], colpayload[i]); state.entries[index].value.active++; } } else // is a retraction { heldState.state = this.deaccumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active--; } } batch.Release(); batch.Return(); }
public unsafe void OnNext(StreamMessage <TInnerKey, TSource> batch) { var batches = new StreamMessage <TInnerKey, TSource> [this.totalBranchesL2]; int[] counts = new int[this.totalBranchesL2]; for (int i = 0; i < this.totalBranchesL2; i++) { this.l1Pool.Get(out batches[i]); batches[i].CloneFrom(batch); batches[i].bitvector.ReturnClear(); this.resetBV.IncrementRefCount(1); batches[i].bitvector = this.resetBV.MakeWritable(this.l1Pool.bitvectorPool); } var count = batch.Count; fixed(long *src_bv = batch.bitvector.col) fixed(int *src_hash = batch.hash.col) { if (this.destinationSelectorCompiled == null) { if (this.powerOf2) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] < 0) { AddPunctuationOrLowWatermarkToAllBatches( batches, batch.vsync.col[i], batch.vother.col[i], batch.key.col[i], batch.hash.col[i]); } continue; } var index = src_hash[i] & this.totalBranchesL2Mask; batches[index].bitvector.col[i >> 6] &= ~(1L << (i & 0x3f)); // pass only along that branch counts[index] = i + 1; } } else { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] < 0) { AddPunctuationOrLowWatermarkToAllBatches( batches, batch.vsync.col[i], batch.vother.col[i], batch.key.col[i], batch.hash.col[i]); } continue; } var index = (src_hash[i] & 0x7fffffff) % this.totalBranchesL2; batches[index].bitvector.col[i >> 6] &= ~(1L << (i & 0x3f)); // pass only along that branch counts[index] = i + 1; } } } else { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] < 0) { AddPunctuationOrLowWatermarkToAllBatches( batches, batch.vsync.col[i], batch.vother.col[i], batch.key.col[i], batch.hash.col[i]); } continue; } var dest = this.destinationSelectorCompiled(batch.key.col[i], src_hash[i], batch.payload.col[i]); for (int j = 0; j < dest.Length; j++) { batches[j].bitvector.col[i >> 6] &= ~(1L << (i & 0x3f)); // pass only along chosen branches counts[j] = i + 1; } } } } batch.Release(); batch.Return(); for (int i = 0; i < this.totalBranchesL2; i++) { batches[i].Count = counts[i]; if (counts[i] > 0) { this.Observers[i].OnNext(batches[i]); } else { batches[i].Free(); } } }
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(long *col_bv = batch.bitvector.col) { var colpayload = batch.payload.col; for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == long.MinValue) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var colkey_i = this.keySelector(colpayload[i]); var col_hash_i = this.keyComparerGetHashCode(colkey_i); var syncTime = col_vsync[i]; bool cachedState = false; // Handle time moving forward if (syncTime > this.lastSyncTime) { /* Issue start edges for held aggregates */ if (this.currentState != null && this.heldAggregates.Count == 1) { // there is just one held aggregate, and currentState is set // so currentState has to be the held aggregate cachedState = true; if (this.currentState.active > 0) { int c = this.batch.Count; this.batch.vsync.col[c] = this.currentState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else { this.aggregateByKey.Remove(this.currentKey, this.currentHash); this.currentState = null; } } else { int iter1 = FastDictionary <TKey, HeldState <TState> > .IteratorStart; while (this.heldAggregates.Iterate(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; if (iter1entry.value.active > 0) { int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.finalResultSelector(iter1entry.key, this.computeResult(iter1entry.value.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else { this.aggregateByKey.Remove(iter1entry.key); // , (currentKey, currentHash); this.currentState = null; } } // Time has moved forward, clear the held aggregates this.heldAggregates.Clear(); this.currentState = null; } // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } if (this.currentState == null || ((!this.isUngrouped) && (this.currentHash != col_hash_i || !this.keyComparerEquals(this.currentKey, colkey_i)))) { if (cachedState) { cachedState = false; this.heldAggregates.Clear(); } // Need to retrieve the key from the dictionary this.currentKey = colkey_i; this.currentHash = col_hash_i; if (!this.heldAggregates.Lookup(this.currentKey, this.currentHash, out int index)) { // First time group is active for this time if (!this.aggregateByKey.Lookup(this.currentKey, this.currentHash, out int aggindex)) { // New group. Create new state this.currentState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.aggregateByKey.Insert(this.currentKey, this.currentState, this.currentHash); // No output because initial state is empty } else { this.currentState = this.aggregateByKey.entries[aggindex].value; if (syncTime > this.currentState.timestamp) { if (this.currentState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.currentState.timestamp = syncTime; } } this.heldAggregates.Insert(ref index, this.currentKey, this.currentState); } else { // read new currentState from _heldAgg index this.currentState = this.heldAggregates.entries[index].value; } } else { if (syncTime > this.currentState.timestamp) { if (this.currentState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.currentState.timestamp = syncTime; } } if (col_vsync[i] < col_vother[i]) // insert event { this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active++; } else // is a retraction { this.currentState.state = this.deaccumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active--; } } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; HeldState <TState> heldState; // Handle time moving forward if (syncTime > this.lastSyncTime) { foreach (int iter1 in this.heldAggregates) { var iter1entry = this.aggregateByKey.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.computeResult(iter1entry.value.state); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else { // Update instance of key in case consumer tracks lifetime of the key object. // Otherwise it may live past the Window lifetime. this.aggregateByKey.entries[aggindex].key = colkey[i]; if (this.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } } // It's always a start edge heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { OnLowWatermark(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = PartitionedStreamEvent.LowWatermarkOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else if (col_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation var partitionKey = this.getPartitionKey(colkey[i]); bool emitPunctuation = false; PartitionEntry partitionEntry; if (!this.partitionData.Lookup(partitionKey, out int partitionKeyIndex)) { this.partitionData.Insert(partitionKey, partitionEntry = new PartitionEntry { lastSyncTime = col_vsync[i], heldAggregates = this.dictionaryGenerator() }); emitPunctuation = true; } else { partitionEntry = this.partitionData.entries[partitionKeyIndex].value; emitPunctuation = partitionEntry.lastSyncTime < col_vsync[i]; } OnPunctuation(partitionEntry, col_vsync[i]); if (emitPunctuation) { int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = col_hash[i]; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } } continue; } var syncTime = col_vsync[i]; var partition = this.getPartitionKey(colkey[i]); PartitionEntry entry; // Handle time moving forward if (!this.partitionData.Lookup(partition, out int partitionIndex)) { this.partitionData.Insert(partition, entry = new PartitionEntry { lastSyncTime = syncTime, heldAggregates = this.dictionaryGenerator() }); } else if (syncTime > (entry = this.partitionData.entries[partitionIndex].value).lastSyncTime) { int iter1 = FastDictionary <TKey, TState> .IteratorStart; while (entry.heldAggregates.Iterate(ref iter1)) { var iter1entry = entry.heldAggregates.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = entry.lastSyncTime; this.batch.vother.col[c] = entry.lastSyncTime + this.hop; this.batch.payload.col[c] = this.computeResult(iter1entry.value); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } if (hasDisposableState) { ((IDisposable)iter1entry.value).Dispose(); } } entry.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime entry.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!entry.heldAggregates.Lookup(colkey[i], col_hash[i], out int index)) { entry.heldAggregates.Insert(ref index, colkey[i], this.initialState()); } var entries = entry.heldAggregates.entries; entries[index].value = this.accumulate(entries[index].value, col_vsync[i], colpayload[i]); } } batch.Release(); batch.Return(); }