public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch) { this.pool.Get(out StreamMessage <TKey, TResult> outputBatch); var count = batch.Count; outputBatch.vsync = batch.vsync; outputBatch.vother = batch.vother; outputBatch.key = batch.key; outputBatch.hash = batch.hash; outputBatch.iter = batch.iter; this.pool.GetPayload(out outputBatch.payload); outputBatch.bitvector = batch.bitvector; var dest = outputBatch.payload.col; var src = batch.payload.col; fixed(long *bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((bv[i >> 6] & (1L << (i & 0x3f))) == 0) { dest[i] = this.selectorFunc(src[i]); } } } outputBatch.Count = count; batch.payload.Return(); batch.Return(); this.Observer.OnNext(outputBatch); }
private void CheckAndRelease(StreamMessage <Empty, TPayload> batch) { if (batch != null) { batch.Release(); batch.Return(); } }
public unsafe void OnNext(StreamMessage <TOuterKey, TSource> batch) { this.l1Pool.Get(out StreamMessage <CompoundGroupKey <TOuterKey, TInnerKey>, TSource> outputBatch); outputBatch.vsync = batch.vsync; outputBatch.vother = batch.vother; outputBatch.payload = batch.payload; outputBatch.hash = batch.hash.MakeWritable(this.l1Pool.intPool); outputBatch.bitvector = batch.bitvector; this.l1Pool.GetKey(out outputBatch.key); outputBatch.Count = batch.Count; var count = batch.Count; var srckey = batch.key.col; var destkey = outputBatch.key.col; var destpayload = outputBatch; fixed(long *srcbv = batch.bitvector.col) { fixed(int *desthash = outputBatch.hash.col) { for (int i = 0; i < count; i++) { if ((srcbv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] == long.MinValue) { destkey[i].outerGroup = srckey[i]; destkey[i].innerGroup = default; desthash[i] = batch.hash.col[i]; outputBatch.bitvector.col[i >> 6] |= (1L << (i & 0x3f)); } continue; } var key = this.keySelectorFunc(destpayload[i]); var innerHash = this.innerHashCode(key); var hash = desthash[i] ^ innerHash; destkey[i].outerGroup = srckey[i]; destkey[i].innerGroup = key; destkey[i].hashCode = hash; desthash[i] = hash; } } } batch.key.Return(); batch.Return(); this.Observer.OnNext(outputBatch); }
public unsafe void OnNext(StreamMessage <Empty, TPayload> batch) { this.l1Pool.Get(out StreamMessage <PartitionKey <TPartitionKey>, TPayload> outputBatch); if (this.partitionLag > 0) { outputBatch.vsync = batch.vsync.MakeWritable(this.l1Pool.longPool); } else { outputBatch.vsync = batch.vsync; } outputBatch.vother = batch.vother.MakeWritable(this.l1Pool.longPool); outputBatch.payload = batch.payload; outputBatch.hash = batch.hash.MakeWritable(this.l1Pool.intPool); outputBatch.bitvector = batch.bitvector; this.l1Pool.GetKey(out outputBatch.key); outputBatch.Count = batch.Count; var count = batch.Count; var destKey = outputBatch.key.col; var destPayload = outputBatch; fixed(int *destHash = outputBatch.hash.col) { for (int i = 0; i < count; i++) { if ((batch.bitvector.col[i >> 6] & (1L << (i & 0x3f))) != 0 && batch.vother.col[i] == StreamEvent.PunctuationOtherTime) { if (this.partitionLag > 0) { outputBatch.vsync.col[i] = batch.vsync.col[i] - this.partitionLag; } outputBatch.vother.col[i] = PartitionedStreamEvent.LowWatermarkOtherTime; continue; } var key = this.keySelectorFunc(destPayload[i]); destKey[i] = new PartitionKey <TPartitionKey> { Key = key }; destHash[i] = key.GetHashCode(); } } batch.key.Return(); batch.Return(); this.Observer.OnNext(outputBatch); }
public unsafe void OnNext(StreamMessage <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult> batch) { outPool.Get(out StreamMessage <TOuterKey, TResult> tmp); tmp.AllocatePayload(); outPool.GetKey(out tmp.key); tmp.hash = batch.hash.MakeWritable(outPool.intPool); var count = batch.Count; tmp.vsync = batch.vsync; tmp.vother = batch.vother; tmp.bitvector = batch.bitvector; fixed(long *srcbv = batch.bitvector.col) fixed(int *desthash = tmp.hash.col) { var srckey = batch.key.col; var destkey = tmp.key.col; for (int i = 0; i < count; i++) { if ((srcbv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] == long.MinValue) { destkey[i] = srckey[i].outerGroup; desthash[i] = batch.hash.col[i]; tmp.bitvector.col[i >> 6] |= 1L << (i & 0x3f); } continue; } destkey[i] = srckey[i].outerGroup; tmp[i] = resultSelector(srckey[i].innerGroup, batch[i]); desthash[i] = this.outerHashCode(destkey[i]); } } tmp.Count = count; tmp.Seal(); Observer.OnNext(tmp); batch.ReleasePayload(); batch.key.Return(); batch.Return(); }
public unsafe void OnNext(StreamMessage <TInnerKey, TInnerResult> batch) { outPool.Get(out StreamMessage <Empty, TResult> tmp); tmp.AllocatePayload(); outPool.GetKey(out tmp.key); tmp.hash = batch.hash.MakeWritable(outPool.intPool); var count = batch.Count; tmp.vsync = batch.vsync; tmp.vother = batch.vother; tmp.bitvector = batch.bitvector; fixed(long *srcbv = batch.bitvector.col) fixed(int *desthash = tmp.hash.col) { var srckey = batch.key.col; var destkey = tmp.key.col; Array.Clear(tmp.hash.col, 0, count); var unit = Empty.Default; for (int i = 0; i < count; i++) { if ((srcbv[i >> 6] & (1L << (i & 0x3f))) != 0) { continue; } destkey[i] = unit; tmp[i] = resultSelector(srckey[i], batch[i]); } } tmp.Count = count; tmp.Seal(); Observer.OnNext(tmp); batch.ReleasePayload(); batch.key.Return(); batch.Return(); }
public unsafe void OnNext(StreamMessage <TOuterKey, TSource> batch) { this.l1Pool.Get(out StreamMessage <TInnerKey, TSource> outputBatch); outputBatch.vsync = batch.vsync; outputBatch.vother = batch.vother; outputBatch.payload = batch.payload; outputBatch.hash = batch.hash.MakeWritable(this.l1Pool.intPool); outputBatch.bitvector = batch.bitvector; this.l1Pool.GetKey(out outputBatch.key); outputBatch.Count = batch.Count; var count = batch.Count; var destkey = outputBatch.key.col; var destpayload = outputBatch; fixed(long *srcbv = batch.bitvector.col) { fixed(int *desthash = outputBatch.hash.col) { for (int i = 0; i < count; i++) { if ((srcbv[i >> 6] & (1L << (i & 0x3f))) != 0) { continue; } var key = this.keySelectorFunc(destpayload[i]); destkey[i] = key; desthash[i] = this.innerHashCode(key); } } } batch.key.Return(); batch.Return(); this.Observer.OnNext(outputBatch); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { AdvanceTime(syncTime); } // Need to retrieve the key from the dictionary HeldState <TState> heldState; if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else if (this.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { if (heldState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { this.batch.iter = batch.iter; FlushContents(); this.batch.iter = batch.iter; } } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } if (col_vsync[i] < col_vother[i]) // insert event { heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active++; // Update ECQ if (col_vother[i] < StreamEvent.InfinitySyncTime) { FastDictionary <TKey, StateAndActive <TState> > state; int index; if (this.ecq.Count > 0) { if (!this.ecq.TryGetValue(col_vother[i], out state)) { this.ecqEntryPool.Get(out state); state.Lookup(colkey[i], col_hash[i], out index); state.Insert(ref index, colkey[i], new StateAndActive <TState> { state = this.initialState() }); this.ecq.Add(col_vother[i], state); } else { if (!state.Lookup(colkey[i], col_hash[i], out index)) { state.Insert(ref index, colkey[i], new StateAndActive <TState> { state = this.initialState() }); } } } else { this.ecqEntryPool.Get(out state); state.Lookup(colkey[i], col_hash[i], out index); state.Insert(ref index, colkey[i], new StateAndActive <TState> { state = this.initialState() }); this.ecq.Add(col_vother[i], state); } state.entries[index].value.state = this.accumulate(state.entries[index].value.state, col_vsync[i], colpayload[i]); state.entries[index].value.active++; } } else // is a retraction { heldState.state = this.deaccumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active--; } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { OnLowWatermark(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = PartitionedStreamEvent.LowWatermarkOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else if (col_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation var p = this.getPartitionKey(colkey[i]); PartitionEntry partitionEntry; if (!this.partitionData.Lookup(p, out int partitionIndex)) { this.partitionData.Insert(p, partitionEntry = new PartitionEntry { lastSyncTime = col_vsync[i], ecq = new CircularBuffer <EcqState>(this.hopsPerDuration) }); } else { partitionEntry = this.partitionData.entries[partitionIndex].value; } OnPunctuation(partitionEntry, col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; var partition = this.getPartitionKey(colkey[i]); HeldState <TState> heldState; PartitionEntry entry; // Handle time moving forward if (!this.partitionData.Lookup(partition, out int pIndex)) { this.partitionData.Insert(partition, entry = new PartitionEntry { lastSyncTime = syncTime, ecq = new CircularBuffer <EcqState>(this.hopsPerDuration) }); } else if (syncTime > (entry = this.partitionData.entries[pIndex].value).lastSyncTime) { AdvanceTime(entry, syncTime); } // Need to retrieve the key from the dictionary if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; entry.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else if (entry.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { if (heldState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { this.batch.iter = batch.iter; FlushContents(); this.batch.iter = batch.iter; } } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } if (col_vsync[i] < col_vother[i]) // insert event { heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active++; // Update ECQ if (col_vother[i] < StreamEvent.InfinitySyncTime) { EcqState state; StateAndActive <TState> newState; int index; if (entry.ecq.Count > 0) { state = entry.ecq.PeekLast(); if (state.timestamp != col_vother[i]) { state = new EcqState { timestamp = col_vother[i] }; this.ecqEntryPool.Get(out state.states); newState = new StateAndActive <TState> { state = this.initialState() }; state.states.Lookup(colkey[i], col_hash[i], out index); state.states.Insert(ref index, colkey[i], newState); entry.ecq.Enqueue(ref state); } else { if (!state.states.Lookup(colkey[i], col_hash[i], out index)) { newState = new StateAndActive <TState> { state = this.initialState() }; state.states.Insert(ref index, colkey[i], newState); } else { newState = state.states.entries[index].value; } } } else { state = new EcqState { timestamp = col_vother[i] }; this.ecqEntryPool.Get(out state.states); newState = new StateAndActive <TState> { state = this.initialState() }; state.states.Lookup(colkey[i], col_hash[i], out index); state.states.Insert(ref index, colkey[i], newState); entry.ecq.Enqueue(ref state); } newState.state = this.accumulate(newState.state, col_vsync[i], colpayload[i]); newState.active++; } } else // is a retraction { heldState.state = this.deaccumulate(heldState.state, col_vsync[i], colpayload[i]); heldState.active--; } } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { OnLowWatermark(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = PartitionedStreamEvent.LowWatermarkOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else if (col_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation var p = this.getPartitionKey(colkey[i]); PartitionEntry partitionEntry; if (!this.partitionData.Lookup(p, out int partitionIndex)) { this.partitionData.Insert(p, partitionEntry = new PartitionEntry { lastSyncTime = col_vsync[i] }); } else { partitionEntry = this.partitionData.entries[partitionIndex].value; } OnPunctuation(partitionEntry, col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; var partition = this.getPartitionKey(colkey[i]); HeldState <TState> heldState; PartitionEntry entry; // Handle time moving forward if (!this.partitionData.Lookup(partition, out int pIndex)) { this.partitionData.Insert(partition, entry = new PartitionEntry { lastSyncTime = syncTime }); } else if (syncTime > (entry = this.partitionData.entries[pIndex].value).lastSyncTime) { /* Issue start edges for held aggregates */ foreach (int iter1 in entry.heldAggregates) { var iter1entry = this.aggregateByKey.entries[iter1]; if (partition.Equals(this.getPartitionKey(iter1entry.key))) { int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.computeResult(iter1entry.value.state); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } } // Time has moved forward, clear the held aggregates entry.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime entry.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; entry.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else if (entry.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch) { var count = batch.Count; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { if (this.currentState != null && this.held) { int c = this.batch.Count; this.batch.vsync.col[c] = this.currentState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.computeResult(this.currentState.state); this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } this.held = false; } // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } if (this.currentState == null) { this.currentState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.held = true; } else { if (syncTime > this.currentState.timestamp) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.computeResult(this.currentState.state); this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } this.currentState.timestamp = syncTime; this.held = true; } } // It's always a start edge this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]); } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { AdvanceTime(syncTime); } if (this.currentState == null) { this.currentEcqHeldState = null; this.currentState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.held = true; // No output because initial state is empty } else { if (syncTime > this.currentState.timestamp) { if (this.currentState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.computeResult(this.currentState.state); this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.currentState.timestamp = syncTime; this.held = true; } } if (col_vsync[i] < col_vother[i]) // insert event { this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active++; // Update ECQ if (col_vother[i] < StreamEvent.InfinitySyncTime) { if ((this.currentEcqHeldState == null) || (this.currentEcqHeldState.timestamp != col_vother[i])) { if (this.ecq.Count > 0) { this.currentEcqHeldState = this.ecq.PeekLast(); if (this.currentEcqHeldState.timestamp != col_vother[i]) { this.currentEcqHeldState = new HeldState <TState> { state = this.initialState(), timestamp = col_vother[i] }; this.ecq.Enqueue(ref this.currentEcqHeldState); } } else { this.currentEcqHeldState = new HeldState <TState> { state = this.initialState(), timestamp = col_vother[i] }; this.ecq.Enqueue(ref this.currentEcqHeldState); } } this.currentEcqHeldState.state = this.accumulate(this.currentEcqHeldState.state, col_vsync[i], colpayload[i]); this.currentEcqHeldState.active++; } } else // is a retraction { this.currentState.state = this.deaccumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active--; } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { int iter1 = FastDictionary <TKey, TState> .IteratorStart; while (this.heldAggregates.Iterate(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = this.lastSyncTime; this.batch.vother.col[c] = this.lastSyncTime + this.hop; this.batch.payload.col[c] = this.computeResult(iter1entry.value); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = iter1entry.key.GetHashCode(); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } if (hasDisposableState) { DisposeStateLocal(); } this.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!this.heldAggregates.Lookup(colkey[i], col_hash[i], out int index)) { this.heldAggregates.Insert(ref index, colkey[i], this.initialState()); } var entries = this.heldAggregates.entries; entries[index].value = this.accumulate(entries[index].value, col_vsync[i], colpayload[i]); } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == StreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var syncTime = col_vsync[i]; HeldState <TState> heldState; // Handle time moving forward if (syncTime > this.lastSyncTime) { foreach (int iter1 in this.heldAggregates) { var iter1entry = this.aggregateByKey.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.computeResult(iter1entry.value.state); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex)) { // New group. Create new state heldState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i])); // No output because initial state is empty } else { // Update instance of key in case consumer tracks lifetime of the key object. // Otherwise it may live past the Window lifetime. this.aggregateByKey.entries[aggindex].key = colkey[i]; if (this.heldAggregates.Add(aggindex)) { // First time group is active for this time heldState = this.aggregateByKey.entries[aggindex].value; if (syncTime > heldState.timestamp) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = heldState.timestamp; this.batch.payload.col[c] = this.computeResult(heldState.state); this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } heldState.timestamp = syncTime; } } else { // read new currentState from _heldAgg index heldState = this.aggregateByKey.entries[aggindex].value; } } // It's always a start edge heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]); } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch) { var count = batch.Count; var colkey = batch.key.col; var colpayload = batch.payload.col; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(int *col_hash = batch.hash.col) fixed(long *col_bv = batch.bitvector.col) { for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == PartitionedStreamEvent.LowWatermarkOtherTime) { OnLowWatermark(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = PartitionedStreamEvent.LowWatermarkOtherTime; this.batch.key.col[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else if (col_vother[i] == PartitionedStreamEvent.PunctuationOtherTime) { // We have found a row that corresponds to punctuation var partitionKey = this.getPartitionKey(colkey[i]); bool emitPunctuation = false; PartitionEntry partitionEntry; if (!this.partitionData.Lookup(partitionKey, out int partitionKeyIndex)) { this.partitionData.Insert(partitionKey, partitionEntry = new PartitionEntry { lastSyncTime = col_vsync[i], heldAggregates = this.dictionaryGenerator() }); emitPunctuation = true; } else { partitionEntry = this.partitionData.entries[partitionKeyIndex].value; emitPunctuation = partitionEntry.lastSyncTime < col_vsync[i]; } OnPunctuation(partitionEntry, col_vsync[i]); if (emitPunctuation) { int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = colkey[i]; this.batch.hash.col[c] = col_hash[i]; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } } continue; } var syncTime = col_vsync[i]; var partition = this.getPartitionKey(colkey[i]); PartitionEntry entry; // Handle time moving forward if (!this.partitionData.Lookup(partition, out int partitionIndex)) { this.partitionData.Insert(partition, entry = new PartitionEntry { lastSyncTime = syncTime, heldAggregates = this.dictionaryGenerator() }); } else if (syncTime > (entry = this.partitionData.entries[partitionIndex].value).lastSyncTime) { int iter1 = FastDictionary <TKey, TState> .IteratorStart; while (entry.heldAggregates.Iterate(ref iter1)) { var iter1entry = entry.heldAggregates.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = entry.lastSyncTime; this.batch.vother.col[c] = entry.lastSyncTime + this.hop; this.batch.payload.col[c] = this.computeResult(iter1entry.value); this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = this.keyComparerGetHashCode(iter1entry.key); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } if (hasDisposableState) { ((IDisposable)iter1entry.value).Dispose(); } } entry.heldAggregates.Clear(); // Since sync time changed, set lastSyncTime entry.lastSyncTime = syncTime; } // Need to retrieve the key from the dictionary if (!entry.heldAggregates.Lookup(colkey[i], col_hash[i], out int index)) { entry.heldAggregates.Insert(ref index, colkey[i], this.initialState()); } var entries = entry.heldAggregates.entries; entries[index].value = this.accumulate(entries[index].value, col_vsync[i], colpayload[i]); } } batch.Release(); batch.Return(); }
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch) { this.batch.iter = batch.iter; var count = batch.Count; fixed(long *col_vsync = batch.vsync.col) fixed(long *col_vother = batch.vother.col) fixed(long *col_bv = batch.bitvector.col) { var colpayload = batch.payload.col; for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == long.MinValue) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = Empty.Default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f)); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } continue; } var colkey_i = this.keySelector(colpayload[i]); var col_hash_i = this.keyComparerGetHashCode(colkey_i); var syncTime = col_vsync[i]; bool cachedState = false; // Handle time moving forward if (syncTime > this.lastSyncTime) { /* Issue start edges for held aggregates */ if (this.currentState != null && this.heldAggregates.Count == 1) { // there is just one held aggregate, and currentState is set // so currentState has to be the held aggregate cachedState = true; if (this.currentState.active > 0) { int c = this.batch.Count; this.batch.vsync.col[c] = this.currentState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else { this.aggregateByKey.Remove(this.currentKey, this.currentHash); this.currentState = null; } } else { int iter1 = FastDictionary <TKey, HeldState <TState> > .IteratorStart; while (this.heldAggregates.Iterate(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; if (iter1entry.value.active > 0) { int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; this.batch.payload.col[c] = this.finalResultSelector(iter1entry.key, this.computeResult(iter1entry.value.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } else { this.aggregateByKey.Remove(iter1entry.key); // , (currentKey, currentHash); this.currentState = null; } } // Time has moved forward, clear the held aggregates this.heldAggregates.Clear(); this.currentState = null; } // Since sync time changed, set lastSyncTime this.lastSyncTime = syncTime; } if (this.currentState == null || ((!this.isUngrouped) && (this.currentHash != col_hash_i || !this.keyComparerEquals(this.currentKey, colkey_i)))) { if (cachedState) { cachedState = false; this.heldAggregates.Clear(); } // Need to retrieve the key from the dictionary this.currentKey = colkey_i; this.currentHash = col_hash_i; if (!this.heldAggregates.Lookup(this.currentKey, this.currentHash, out int index)) { // First time group is active for this time if (!this.aggregateByKey.Lookup(this.currentKey, this.currentHash, out int aggindex)) { // New group. Create new state this.currentState = new HeldState <TState> { state = this.initialState(), timestamp = syncTime }; this.aggregateByKey.Insert(this.currentKey, this.currentState, this.currentHash); // No output because initial state is empty } else { this.currentState = this.aggregateByKey.entries[aggindex].value; if (syncTime > this.currentState.timestamp) { if (this.currentState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.currentState.timestamp = syncTime; } } this.heldAggregates.Insert(ref index, this.currentKey, this.currentState); } else { // read new currentState from _heldAgg index this.currentState = this.heldAggregates.entries[index].value; } } else { if (syncTime > this.currentState.timestamp) { if (this.currentState.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = this.currentState.timestamp; this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state)); this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { FlushContents(); } } this.currentState.timestamp = syncTime; } } if (col_vsync[i] < col_vother[i]) // insert event { this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active++; } else // is a retraction { this.currentState.state = this.deaccumulate(this.currentState.state, col_vsync[i], colpayload[i]); this.currentState.active--; } } } batch.Release(); batch.Return(); }
public unsafe void OnNext(StreamMessage <TInnerKey, TSource> batch) { var batches = new StreamMessage <TInnerKey, TSource> [this.totalBranchesL2]; int[] counts = new int[this.totalBranchesL2]; for (int i = 0; i < this.totalBranchesL2; i++) { this.l1Pool.Get(out batches[i]); batches[i].CloneFrom(batch); batches[i].bitvector.ReturnClear(); this.resetBV.IncrementRefCount(1); batches[i].bitvector = this.resetBV.MakeWritable(this.l1Pool.bitvectorPool); } var count = batch.Count; fixed(long *src_bv = batch.bitvector.col) fixed(int *src_hash = batch.hash.col) { if (this.destinationSelectorCompiled == null) { if (this.powerOf2) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] < 0) { AddPunctuationOrLowWatermarkToAllBatches( batches, batch.vsync.col[i], batch.vother.col[i], batch.key.col[i], batch.hash.col[i]); } continue; } var index = src_hash[i] & this.totalBranchesL2Mask; batches[index].bitvector.col[i >> 6] &= ~(1L << (i & 0x3f)); // pass only along that branch counts[index] = i + 1; } } else { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] < 0) { AddPunctuationOrLowWatermarkToAllBatches( batches, batch.vsync.col[i], batch.vother.col[i], batch.key.col[i], batch.hash.col[i]); } continue; } var index = (src_hash[i] & 0x7fffffff) % this.totalBranchesL2; batches[index].bitvector.col[i >> 6] &= ~(1L << (i & 0x3f)); // pass only along that branch counts[index] = i + 1; } } } else { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (batch.vother.col[i] < 0) { AddPunctuationOrLowWatermarkToAllBatches( batches, batch.vsync.col[i], batch.vother.col[i], batch.key.col[i], batch.hash.col[i]); } continue; } var dest = this.destinationSelectorCompiled(batch.key.col[i], src_hash[i], batch.payload.col[i]); for (int j = 0; j < dest.Length; j++) { batches[j].bitvector.col[i >> 6] &= ~(1L << (i & 0x3f)); // pass only along chosen branches counts[j] = i + 1; } } } } batch.Release(); batch.Return(); for (int i = 0; i < this.totalBranchesL2; i++) { batches[i].Count = counts[i]; if (counts[i] > 0) { this.Observers[i].OnNext(batches[i]); } else { batches[i].Free(); } } }