public override unsafe void OnNext(StreamMessage <Empty, TInput> batch)
{
this.batch.iter = batch.iter;
var count = batch.Count;
fixed(long *col_vsync = batch.vsync.col)
fixed(long *col_vother = batch.vother.col)
fixed(long *col_bv = batch.bitvector.col)
{
var colpayload = batch.payload.col;
for (int i = 0; i < count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == long.MinValue)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = long.MinValue;
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
continue;
}
var colkey_i = this.keySelector(colpayload[i]);
var col_hash_i = this.keyComparerGetHashCode(colkey_i);
var syncTime = col_vsync[i];
bool cachedState = false;
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
/* Issue start edges for held aggregates */
if (this.currentState != null && this.heldAggregates.Count == 1)
{
// there is just one held aggregate, and currentState is set
// so currentState has to be the held aggregate
cachedState = true;
if (this.currentState.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = this.currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state));
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
else
{
this.aggregateByKey.Remove(this.currentKey, this.currentHash);
this.currentState = null;
}
}
else
{
int iter1 = FastDictionary <TKey, HeldState <TState> > .IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
this.batch.payload.col[c] = this.finalResultSelector(iter1entry.key, this.computeResult(iter1entry.value.state));
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
else
{
this.aggregateByKey.Remove(iter1entry.key); // , (currentKey, currentHash);
this.currentState = null;
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
this.currentState = null;
}
// Since sync time changed, set lastSyncTime
this.lastSyncTime = syncTime;
}
if (this.currentState == null || ((!this.isUngrouped) && (this.currentHash != col_hash_i || !this.keyComparerEquals(this.currentKey, colkey_i))))
{
if (cachedState)
{
cachedState = false;
this.heldAggregates.Clear();
}
// Need to retrieve the key from the dictionary
this.currentKey = colkey_i;
this.currentHash = col_hash_i;
if (!this.heldAggregates.Lookup(this.currentKey, this.currentHash, out int index))
{
// First time group is active for this time
if (!this.aggregateByKey.Lookup(this.currentKey, this.currentHash, out int aggindex))
{
// New group. Create new state
this.currentState = new HeldState <TState> {
state = this.initialState(), timestamp = syncTime
};
this.aggregateByKey.Insert(this.currentKey, this.currentState, this.currentHash);
// No output because initial state is empty
}
else
{
this.currentState = this.aggregateByKey.entries[aggindex].value;
if (syncTime > this.currentState.timestamp)
{
if (this.currentState.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = this.currentState.timestamp;
this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state));
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
this.currentState.timestamp = syncTime;
}
}
this.heldAggregates.Insert(ref index, this.currentKey, this.currentState);
}
else
{
// read new currentState from _heldAgg index
this.currentState = this.heldAggregates.entries[index].value;
}
}
else
{
if (syncTime > this.currentState.timestamp)
{
if (this.currentState.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = this.currentState.timestamp;
this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state));
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
this.currentState.timestamp = syncTime;
}
}
if (col_vsync[i] < col_vother[i]) // insert event
{
this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]);
this.currentState.active++;
}
else // is a retraction
{
this.currentState.state = this.deaccumulate(this.currentState.state, col_vsync[i], colpayload[i]);
this.currentState.active--;
}
}
}
batch.Release();
batch.Return();
}
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch)
{
var count = batch.Count;
var colkey = batch.key.col;
var colpayload = batch.payload.col;
fixed(long *col_vsync = batch.vsync.col)
fixed(long *col_vother = batch.vother.col)
fixed(long *col_bv = batch.bitvector.col)
{
for (int i = 0; i < count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == StreamEvent.PunctuationOtherTime)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime;
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
continue;
}
var syncTime = col_vsync[i];
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
if (this.currentState != null)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = this.currentState.timestamp;
this.batch.vother.col[c] = this.currentState.timestamp + this.hop;
this.batch.payload.col[c] = this.computeResult(this.currentState.state);
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
if (hasDisposableState)
{
DisposeStateLocal();
}
this.currentState = null;
}
// Since sync time changed, set lastSyncTime
this.lastSyncTime = syncTime;
}
if (this.currentState == null)
{
this.currentState = new HeldState <TState> {
state = this.initialState(), timestamp = syncTime
}
}
;
else
{
if (syncTime > this.currentState.timestamp)
{
// Reset currentState
if (hasDisposableState)
{
DisposeStateLocal();
}
this.currentState.state = this.initialState();
this.currentState.timestamp = syncTime;
}
}
this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]);
}
}
batch.Release();
batch.Return();
}
public void OnPunctuation(long syncTime)
{
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
/* Issue start edges for held aggregates */
if (this.currentState != null && this.heldAggregates.Count == 1)
{
// there is just one held aggregate, and currentState is set
// so currentState has to be the held aggregate
if (this.currentState.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = this.currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
this.batch.payload.col[c] = this.finalResultSelector(this.currentKey, this.computeResult(this.currentState.state));
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
else
{
this.aggregateByKey.Remove(this.currentKey, this.currentHash);
this.currentState = null;
}
}
else
{
int iter1 = FastDictionary <TKey, HeldState <TState> > .IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
this.batch.payload.col[c] = this.finalResultSelector(iter1entry.key, this.computeResult(iter1entry.value.state));
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
else
{
this.aggregateByKey.Remove(iter1entry.key);
this.currentState = null;
}
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
this.currentState = null;
// Since sync time changed, set lastSyncTime
this.lastSyncTime = syncTime;
}
}
public override unsafe void OnNext(StreamMessage <Empty, TInput> batch)
{
this.batch.iter = batch.iter;
var count = batch.Count;
var colpayload = batch.payload.col;
fixed(long *col_vsync = batch.vsync.col)
fixed(long *col_vother = batch.vother.col)
fixed(int *col_hash = batch.hash.col)
fixed(long *col_bv = batch.bitvector.col)
{
for (int i = 0; i < count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == StreamEvent.PunctuationOtherTime)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime;
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
continue;
}
var syncTime = col_vsync[i];
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
AdvanceTime(syncTime);
}
if (this.currentState == null)
{
this.currentEcqHeldState = null;
this.currentState = new HeldState <TState> {
state = this.initialState(), timestamp = syncTime
};
this.held = true;
// No output because initial state is empty
}
else
{
if (syncTime > this.currentState.timestamp)
{
if (this.currentState.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = this.currentState.timestamp;
this.batch.payload.col[c] = this.computeResult(this.currentState.state);
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
this.currentState.timestamp = syncTime;
this.held = true;
}
}
if (col_vsync[i] < col_vother[i]) // insert event
{
this.currentState.state = this.accumulate(this.currentState.state, col_vsync[i], colpayload[i]);
this.currentState.active++;
// Update ECQ
if (col_vother[i] < StreamEvent.InfinitySyncTime)
{
if ((this.currentEcqHeldState == null) || (this.currentEcqHeldState.timestamp != col_vother[i]))
{
if (this.ecq.Count > 0)
{
this.currentEcqHeldState = this.ecq.PeekLast();
if (this.currentEcqHeldState.timestamp != col_vother[i])
{
this.currentEcqHeldState = new HeldState <TState> {
state = this.initialState(), timestamp = col_vother[i]
};
this.ecq.Enqueue(ref this.currentEcqHeldState);
}
}
else
{
this.currentEcqHeldState = new HeldState <TState> {
state = this.initialState(), timestamp = col_vother[i]
};
this.ecq.Enqueue(ref this.currentEcqHeldState);
}
}
this.currentEcqHeldState.state = this.accumulate(this.currentEcqHeldState.state, col_vsync[i], colpayload[i]);
this.currentEcqHeldState.active++;
}
}
else // is a retraction
{
this.currentState.state = this.deaccumulate(this.currentState.state, col_vsync[i], colpayload[i]);
this.currentState.active--;
}
}
}
batch.Release();
batch.Return();
}
private void AdvanceTime(long syncTime)
{
/* Issue start edges for held aggregates */
if (this.currentState != null && this.held)
{
if (this.currentState.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = this.currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
this.batch.payload.col[c] = this.computeResult(this.currentState.state);
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
this.held = false;
}
/* Process the ECQ up until the new sync time */
while (this.ecq.Count > 0 && this.ecq.PeekFirst().timestamp <= syncTime)
{
HeldState <TState> ecqState = this.ecq.Dequeue();
if (this.currentState.active > 0)
{
// Issue end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = this.currentState.timestamp;
this.batch.payload.col[c] = this.computeResult(this.currentState.state);
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
// Update aggregate
this.currentState.state = this.difference(this.currentState.state, ecqState.state);
this.currentState.active -= ecqState.active;
(ecqState.state as IDisposable)?.Dispose();
if (ecqState.timestamp < syncTime)
{
if (this.currentState.active > 0)
{
// Issue start edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
this.batch.payload.col[c] = this.computeResult(this.currentState.state);
this.batch.key.col[c] = Empty.Default;
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
else
{
(this.currentState.state as IDisposable)?.Dispose();
this.currentState = null;
}
}
else
{
this.held = true;
}
// Update timestamp
if (this.currentState != null)
{
this.currentState.timestamp = ecqState.timestamp;
}
}
// Since sync time changed, set lastSyncTime
this.lastSyncTime = syncTime;
}
public override unsafe void OnNext(StreamMessage <TKey, TInput> batch)
{
this.batch.iter = batch.iter;
var count = batch.Count;
var colkey = batch.key.col;
var colpayload = batch.payload.col;
fixed(long *col_vsync = batch.vsync.col)
fixed(long *col_vother = batch.vother.col)
fixed(int *col_hash = batch.hash.col)
fixed(long *col_bv = batch.bitvector.col)
{
for (int i = 0; i < count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == StreamEvent.PunctuationOtherTime)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = StreamEvent.PunctuationOtherTime;
this.batch.key.col[c] = default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= (1L << (c & 0x3f));
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
continue;
}
var syncTime = col_vsync[i];
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
AdvanceTime(syncTime);
}
// Need to retrieve the key from the dictionary
HeldState <TState> heldState;
if (!this.aggregateByKey.Lookup(colkey[i], col_hash[i], out int aggindex))
{
// New group. Create new state
heldState = new HeldState <TState> {
state = this.initialState(), timestamp = syncTime
};
this.heldAggregates.Add(this.aggregateByKey.Insert(colkey[i], heldState, col_hash[i]));
// No output because initial state is empty
}
else if (this.heldAggregates.Add(aggindex))
{
// First time group is active for this time
heldState = this.aggregateByKey.entries[aggindex].value;
if (syncTime > heldState.timestamp)
{
if (heldState.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = heldState.timestamp;
this.batch.payload.col[c] = this.computeResult(heldState.state);
this.batch.key.col[c] = colkey[i];
this.batch.hash.col[c] = this.keyComparerGetHashCode(colkey[i]);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
FlushContents();
}
}
heldState.timestamp = syncTime;
}
}
else
{
// read new currentState from _heldAgg index
heldState = this.aggregateByKey.entries[aggindex].value;
}
if (col_vsync[i] < col_vother[i]) // insert event
{
heldState.state = this.accumulate(heldState.state, col_vsync[i], colpayload[i]);
heldState.active++;
// Update ECQ
if (col_vother[i] < StreamEvent.InfinitySyncTime)
{
EcqState ecqState;
StateAndActive <TState> newState;
int index;
if (this.ecq.Count > 0)
{
ecqState = this.ecq.PeekLast();
if (ecqState.timestamp != col_vother[i])
{
ecqState = new EcqState {
timestamp = col_vother[i]
};
this.ecqEntryPool.Get(out ecqState.states);
newState = new StateAndActive <TState> {
state = this.initialState()
};
ecqState.states.Lookup(colkey[i], col_hash[i], out index);
ecqState.states.Insert(ref index, colkey[i], newState);
this.ecq.Enqueue(ref ecqState);
}
else
{
if (!ecqState.states.Lookup(colkey[i], col_hash[i], out index))
{
newState = new StateAndActive <TState> {
state = this.initialState()
};
ecqState.states.Insert(ref index, colkey[i], newState);
}
else
{
newState = ecqState.states.entries[index].value;
}
}
}
else
{
ecqState = new EcqState {
timestamp = col_vother[i]
};
this.ecqEntryPool.Get(out ecqState.states);
newState = new StateAndActive <TState> {
state = this.initialState()
};
ecqState.states.Lookup(colkey[i], col_hash[i], out index);
ecqState.states.Insert(ref index, colkey[i], newState);
this.ecq.Enqueue(ref ecqState);
}
newState.state = this.accumulate(newState.state, col_vsync[i], colpayload[i]);
newState.active++;
}
}
else // is a retraction
{
heldState.state = this.deaccumulate(heldState.state, col_vsync[i], colpayload[i]);
heldState.active--;
}
}
}
batch.Release();
batch.Return();
}
