protected override void ProcessLeftBatch(StreamMessage <TKey, TLeft> batch, out bool leftBatchDone, out bool leftBatchFree)
{
leftBatchDone = true;
leftBatchFree = true;
batch.iter = 0;
PooledElasticCircularBuffer <LEntry> queue = null;
TPartitionKey previous = default;
bool first = true;
for (var i = 0; i < batch.Count; i++)
{
if ((batch.bitvector.col[i >> 6] & (1L << (i & 0x3f))) != 0 &&
(batch.vother.col[i] >= 0))
{
continue;
}
if (batch.vother.col[i] == PartitionedStreamEvent.LowWatermarkOtherTime)
{
bool timeAdvanced = this.lastLeftCTI != batch.vsync.col[i];
if (timeAdvanced && this.lastLeftCTI < this.lastRightCTI)
{
this.emitCTI = true;
}
this.lastLeftCTI = batch.vsync.col[i];
foreach (var p in this.seenKeys)
{
this.processQueue.Add(p);
}
continue;
}
var partitionKey = this.getPartitionKey(batch.key.col[i]);
if (first || !partitionKey.Equals(previous))
{
if (this.seenKeys.Add(partitionKey))
{
NewPartition(partitionKey);
}
this.leftQueue.Lookup(partitionKey, out int index);
queue = this.leftQueue.entries[index].value;
this.processQueue.Add(partitionKey);
}
var e = new LEntry
{
Key = batch.key.col[i],
Sync = batch.vsync.col[i],
Other = batch.vother.col[i],
Payload = batch.payload.col[i],
Hash = batch.hash.col[i],
};
queue.Enqueue(ref e);
first = false;
previous = partitionKey;
}
ProcessPendingEntries();
}
public void Return(TKey partitionKey, PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > streamEvents)
{
if (!this.sorters.Lookup(partitionKey, out int index))
{
return;
}
var sorter = this.sorters.entries[index].value;
sorter.Return(streamEvents);
}
private static PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > Merge(
PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > left,
PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > right,
PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > result,
long timestamp)
{
while (true)
{
if ((left.Count == 0) || (left.PeekFirst().SyncTime > timestamp))
{
while ((right.Count > 0) && (right.PeekFirst().SyncTime <= timestamp))
{
var tmp = right.Dequeue();
result.Enqueue(ref tmp);
}
return(result);
}
if ((right.Count == 0) || (right.PeekFirst().SyncTime > timestamp))
{
while ((left.Count > 0) && (left.PeekFirst().SyncTime <= timestamp))
{
var tmp = left.Dequeue();
result.Enqueue(ref tmp);
}
return(result);
}
if (left.PeekFirst().SyncTime < right.PeekFirst().SyncTime)
{
var tmp = left.Dequeue();
result.Enqueue(ref tmp);
}
else
{
var tmp = right.Dequeue();
result.Enqueue(ref tmp);
}
}
}
private void ProcessPendingEntries()
{
foreach (var pKey in this.processQueue)
{
PooledElasticCircularBuffer <LEntry> leftWorking = null;
PooledElasticCircularBuffer <REntry> rightWorking = null;
this.leftQueue.Lookup(pKey, out int index);
leftWorking = this.leftQueue.entries[index].value;
rightWorking = this.rightQueue.entries[index].value;
FastMap <ActiveEvent <TLeft> > leftEdgeMapForPartition = null;
FastMap <ActiveEvent <TRight> > rightEdgeMapForPartition = null;
this.partitionData.Lookup(pKey, out index);
var partition = this.partitionData.entries[index].value;
leftEdgeMapForPartition = partition.leftEdgeMap;
rightEdgeMapForPartition = partition.rightEdgeMap;
while (true)
{
bool hasLeftBatch = leftWorking.TryPeekFirst(out LEntry leftEntry);
bool hasRightBatch = rightWorking.TryPeekFirst(out REntry rightEntry);
FastMap <ActiveEvent <TRight> > .FindTraverser rightEdges = default;
FastMap <ActiveEvent <TLeft> > .FindTraverser leftEdges = default;
if (hasLeftBatch && hasRightBatch)
{
UpdateNextLeftTime(partition, leftEntry.Sync);
UpdateNextRightTime(partition, rightEntry.Sync);
if (partition.nextLeftTime <= partition.nextRightTime)
{
if (leftEntry.Other != long.MinValue)
{
TKey key = leftEntry.Key;
var hash = leftEntry.Hash;
if (rightEdgeMapForPartition.Find(hash, ref rightEdges))
{
while (rightEdges.Next(out int rightIndex))
{
if (this.keyComparer(key, rightEdgeMapForPartition.Values[rightIndex].Key))
{
OutputStartEdge(partition.nextLeftTime, ref key, ref leftEntry.Payload, ref rightEdgeMapForPartition.Values[rightIndex].Payload, hash);
}
}
}
if (!partition.isRightComplete)
{
int newIndex = leftEdgeMapForPartition.Insert(hash);
leftEdgeMapForPartition.Values[newIndex].Populate(ref key, ref leftEntry.Payload);
}
UpdateNextLeftTime(partition, leftEntry.Sync);
}
else
{
OutputPunctuation(leftEntry.Sync, ref leftEntry.Key, leftEntry.Hash);
}
}
else
{
if (rightEntry.Other != long.MinValue)
{
TKey key = rightEntry.Key;
var hash = rightEntry.Hash;
if (leftEdgeMapForPartition.Find(hash, ref leftEdges))
{
while (leftEdges.Next(out int leftIndex))
{
if (this.keyComparer(key, leftEdgeMapForPartition.Values[leftIndex].Key))
{
OutputStartEdge(partition.nextRightTime, ref key, ref leftEdgeMapForPartition.Values[leftIndex].Payload, ref rightEntry.Payload, hash);
}
}
}
if (!partition.isLeftComplete)
{
int newIndex = rightEdgeMapForPartition.Insert(hash);
rightEdgeMapForPartition.Values[newIndex].Populate(ref key, ref rightEntry.Payload);
}
UpdateNextRightTime(partition, rightEntry.Sync);
}
else
{
OutputPunctuation(rightEntry.Sync, ref rightEntry.Key, rightEntry.Hash);
}
}
}
else if (hasLeftBatch)
{
UpdateNextLeftTime(partition, leftEntry.Sync);
if (leftEntry.Other != long.MinValue)
{
TKey key = leftEntry.Key;
var hash = leftEntry.Hash;
if (rightEdgeMapForPartition.Find(hash, ref rightEdges))
{
while (rightEdges.Next(out int rightIndex))
{
if (this.keyComparer(key, rightEdgeMapForPartition.Values[rightIndex].Key))
{
OutputStartEdge(partition.nextLeftTime, ref key, ref leftEntry.Payload, ref rightEdgeMapForPartition.Values[rightIndex].Payload, hash);
}
}
}
if (!partition.isRightComplete)
{
int newIndex = leftEdgeMapForPartition.Insert(hash);
leftEdgeMapForPartition.Values[newIndex].Populate(ref key, ref leftEntry.Payload);
}
UpdateNextLeftTime(partition, leftEntry.Sync);
}
else
{
OutputPunctuation(leftEntry.Sync, ref leftEntry.Key, leftEntry.Hash);
return;
}
}
else if (hasRightBatch)
{
UpdateNextRightTime(partition, rightEntry.Sync);
if (rightEntry.Other != long.MinValue)
{
TKey key = rightEntry.Key;
var hash = rightEntry.Hash;
if (leftEdgeMapForPartition.Find(hash, ref leftEdges))
{
while (leftEdges.Next(out int leftIndex))
{
if (this.keyComparer(key, leftEdgeMapForPartition.Values[leftIndex].Key))
{
OutputStartEdge(partition.nextRightTime, ref key, ref leftEdgeMapForPartition.Values[leftIndex].Payload, ref rightEntry.Payload, hash);
}
}
}
if (!partition.isLeftComplete)
{
int newIndex = rightEdgeMapForPartition.Insert(hash);
rightEdgeMapForPartition.Values[newIndex].Populate(ref key, ref rightEntry.Payload);
}
UpdateNextRightTime(partition, rightEntry.Sync);
}
else
{
OutputPunctuation(rightEntry.Sync, ref rightEntry.Key, rightEntry.Hash);
}
}
else
{
if (partition.nextLeftTime < this.lastLeftCTI)
{
UpdateNextLeftTime(partition, this.lastLeftCTI);
}
if (partition.nextRightTime < this.lastRightCTI)
{
UpdateNextRightTime(partition, this.lastRightCTI);
}
this.cleanKeys.Add(pKey);
break;
}
}
}
if (this.emitCTI)
{
var earliest = Math.Min(this.lastLeftCTI, this.lastRightCTI);
AddLowWatermarkToBatch(earliest);
this.emitCTI = false;
foreach (var p in this.cleanKeys)
{
this.leftQueue.Lookup(p, out int index);
var l = this.leftQueue.entries[index];
var r = this.rightQueue.entries[index];
if (l.value.Count == 0 && r.value.Count == 0)
{
this.seenKeys.Remove(p);
l.value.Dispose();
this.leftQueue.Remove(p);
r.value.Dispose();
this.rightQueue.Remove(p);
}
}
this.cleanKeys.Clear();
}
this.processQueue.Clear();
}
public void Return(PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > streamEvents)
{
this.ecbPool.Return(streamEvents);
}
public void Return(PooledElasticCircularBuffer <StreamEvent <TPayload> > streamEvents)
{
this.ecbPool.Return(streamEvents);
}
private void ProcessPendingEntries()
{
foreach (var workingPartitionKey in this.processQueue)
{
PooledElasticCircularBuffer <Entry> leftWorking = null;
PooledElasticCircularBuffer <Entry> rightWorking = null;
this.leftQueue.Lookup(workingPartitionKey, out int index);
leftWorking = this.leftQueue.entries[index].value;
this.rightQueue.Lookup(workingPartitionKey, out index);
rightWorking = this.rightQueue.entries[index].value;
while (true)
{
bool hasLeftEntry = leftWorking.TryPeekFirst(out Entry leftEntry);
bool hasRightEntry = rightWorking.TryPeekFirst(out Entry rightEntry);
if (hasLeftEntry && hasRightEntry)
{
this.nextLeftTime.Lookup(workingPartitionKey, out int workingIndex);
this.nextLeftTime.entries[workingIndex].value = leftEntry.Sync;
this.nextRightTime.Lookup(workingPartitionKey, out workingIndex);
this.nextRightTime.entries[workingIndex].value = rightEntry.Sync;
switch (leftEntry.Sync.CompareTo(rightEntry.Sync))
{
case -1:
// Output the left tuple
OutputCurrentTuple(leftEntry);
leftWorking.TryDequeue(out leftEntry);
break;
case 0:
// Output both tuples
OutputCurrentTuple(leftEntry);
OutputCurrentTuple(rightEntry);
leftWorking.TryDequeue(out leftEntry);
rightWorking.TryDequeue(out rightEntry);
break;
case 1:
// Output the right tuple
OutputCurrentTuple(rightEntry);
rightWorking.TryDequeue(out rightEntry);
break;
default: throw new InvalidOperationException();
}
}
else if (hasLeftEntry)
{
this.nextLeftTime.Lookup(workingPartitionKey, out int workingIndex);
this.nextLeftTime.entries[workingIndex].value = leftEntry.Sync;
var lastRightTime = !this.nextRightTime.Lookup(workingPartitionKey, out workingIndex)
? long.MinValue
: this.nextRightTime.entries[workingIndex].value;
if (leftEntry.Sync <= Math.Max(lastRightTime, this.lastRightCTI))
{
// Output the left tuple
OutputCurrentTuple(leftEntry);
leftWorking.TryDequeue(out leftEntry);
}
else
{
// Need to wait for the other input
break;
}
}
else if (hasRightEntry)
{
this.nextRightTime.Lookup(workingPartitionKey, out int workingIndex);
this.nextRightTime.entries[workingIndex].value = rightEntry.Sync;
var lastLeftTime = !this.nextLeftTime.Lookup(workingPartitionKey, out workingIndex)
? long.MinValue
: this.nextLeftTime.entries[workingIndex].value;
if (rightEntry.Sync <= Math.Max(lastLeftTime, this.lastLeftCTI))
{
// Output the right tuple
OutputCurrentTuple(rightEntry);
rightWorking.TryDequeue(out rightEntry);
}
else
{
// Need to wait for the other input
break;
}
}
else
{
this.cleanKeys.Add(workingPartitionKey);
break;
}
}
}
if (this.emitCTI)
{
var earliest = Math.Min(this.lastLeftCTI, this.lastRightCTI);
AddLowWatermarkToBatch(earliest);
this.emitCTI = false;
foreach (var p in this.cleanKeys)
{
this.leftQueue.Lookup(p, out int index);
var l = this.leftQueue.entries[index];
var r = this.rightQueue.entries[index];
if (l.value.Count == 0 && r.value.Count == 0)
{
this.seenKeys.Remove(p);
l.value.Dispose();
this.leftQueue.Remove(p);
r.value.Dispose();
this.rightQueue.Remove(p);
}
}
this.cleanKeys.Clear();
}
this.processQueue.Clear();
}