public void DisjointUnionNegativeWatermarkRepro()
{
const int leftKey = 1;
const int rightKey = 2;
var left = new Subject <PartitionedStreamEvent <int, int> >();
var right = new Subject <PartitionedStreamEvent <int, int> >();
var qc = new QueryContainer();
var leftInput = qc.RegisterInput(left);
var rightInput = qc.RegisterInput(right);
var actualOutput = new List <PartitionedStreamEvent <int, int> >();
var inputs = new IStreamable <PartitionKey <int>, int>[] { leftInput, rightInput };
var union = new MultiUnionStreamable <PartitionKey <int>, int>(inputs, guaranteedDisjoint: true);
var egress = qc.RegisterOutput(union).ForEachAsync(o => actualOutput.Add(o));
var process = qc.Restore();
left.OnNext(PartitionedStreamEvent.CreatePoint(leftKey, 100, 1));
right.OnNext(PartitionedStreamEvent.CreatePoint(rightKey, 100, 1));
process.Flush();
left.OnCompleted();
right.OnCompleted();
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(leftKey, 100, 1),
PartitionedStreamEvent.CreatePoint(rightKey, 100, 1),
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(actualOutput));
}
/// <summary>
/// Re-distribute the data across shards
/// </summary>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable with the new key type and values</returns>
public IShardedStreamable <TKey, TPayload> ReDistribute(ILocationDescriptor newLocation = null)
{
int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length;
var shuffleL1Results = new ShuffleSameKeyStreamable <TKey, TPayload, TKey> [this.Streamables.Length];
for (int i = 0; i < this.Streamables.Length; i++)
{
shuffleL1Results[i] = new ShuffleSameKeyStreamable <TKey, TPayload, TKey>(this.Streamables[i], newShardArity, i);
}
var shuffleResults = new IStreamable <TKey, TPayload> [newShardArity];
for (int i = 0; i < newShardArity; i++)
{
shuffleResults[i] = new MultiUnionStreamable <TKey, TPayload>(shuffleL1Results);
}
return(new ShardedStreamable <TKey, TPayload>(shuffleResults));
}
public void DisjointUnionPunctuations()
{
var left = new Subject <StreamEvent <int> >();
var right = new Subject <StreamEvent <int> >();
var qc = new QueryContainer();
var leftInput = qc.RegisterInput(left);
var rightInput = qc.RegisterInput(right);
var actualOutput = new List <StreamEvent <int> >();
var union = new MultiUnionStreamable <Empty, int>(new IStreamable <Empty, int>[] { leftInput, rightInput }, guaranteedDisjoint: true);
var egress = qc.RegisterOutput(union).ForEachAsync(o => actualOutput.Add(o));
var process = qc.Restore();
left.OnNext(StreamEvent.CreatePoint(100, 1));
left.OnNext(StreamEvent.CreatePunctuation <int>(101));
right.OnNext(StreamEvent.CreatePoint(100, 1));
right.OnNext(StreamEvent.CreatePunctuation <int>(110));
process.Flush();
left.OnNext(StreamEvent.CreatePoint(101, 1));
right.OnNext(StreamEvent.CreatePoint(110, 1));
process.Flush();
left.OnCompleted();
right.OnCompleted();
var expected = new StreamEvent <int>[]
{
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePunctuation <int>(101),
StreamEvent.CreatePoint(101, 1),
StreamEvent.CreatePoint(110, 1),
StreamEvent.CreatePunctuation <int>(110),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(actualOutput));
}
/// <summary>
/// Broadcast operation on a sharded streamable
/// </summary>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable post-broadcast</returns>
public IShardedStreamable <TKey, TPayload> Broadcast(ILocationDescriptor newLocation = null)
{
int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length;
var broadcastResults = new SprayGroupImportStreamable <TKey, TPayload> [this.Streamables.Length];
for (int i = 0; i < this.Streamables.Length; i++)
{
broadcastResults[i] = new SprayGroupImportStreamable <TKey, TPayload>(this.Streamables[i], newShardArity, true);
}
var unionResults = new IStreamable <TKey, TPayload> [newShardArity];
for (int i = 0; i < newShardArity; i++)
{
unionResults[i] = new MultiUnionStreamable <TKey, TPayload>(broadcastResults);
}
return(new ShardedStreamable <TKey, TPayload>(unionResults));
}
/// <summary>
/// Multicast operation on a sharded streamable
/// </summary>
/// <param name="destination">A destination desciption for the multicast operation</param>
/// <param name="newLocation">Assign an optional new location descriptor</param>
/// <returns>A new sharded streamable post-multicast</returns>
public IShardedStreamable <TKey, TPayload> Multicast(IDestinationDescriptor destination, ILocationDescriptor newLocation = null)
{
int newShardArity = (newLocation != null && (int)newLocation.GetLocation() > 0) ? (int)newLocation.GetLocation() : this.Streamables.Length;
var destinationSelector = (Expression <Func <TKey, int, TPayload, int[]> >)destination.GetDestination();
var shufflecastL1Results = new ShufflecastStreamable <TPayload, TKey> [this.Streamables.Length];
for (int i = 0; i < this.Streamables.Length; i++)
{
shufflecastL1Results[i] = new ShufflecastStreamable <TPayload, TKey>(null, this.Streamables[i], newShardArity, destinationSelector);
}
var shuffleResults = new IStreamable <TKey, TPayload> [newShardArity];
for (int i = 0; i < newShardArity; i++)
{
shuffleResults[i] = new MultiUnionStreamable <TKey, TPayload>(shufflecastL1Results);
}
return(new ShardedStreamable <TKey, TPayload>(shuffleResults));
}
