private void ProcessInput(
out List <OutOfOrderPartitionedStreamEvent <int, int> > diagnosticEvents,
out List <PartitionedStreamEvent <int, int> > dataEvents)
{
var qc = new QueryContainer();
var ingress = qc.RegisterInput(this.input.ToObservable(), this.disorderPolicy,
PartitionedFlushPolicy.None, PeriodicPunctuationPolicy.None(), this.lowWatermarkPolicy);
var outOfOrderEvents = new List <OutOfOrderPartitionedStreamEvent <int, int> >();
ingress.GetDroppedAdjustedEventsDiagnostic().Subscribe(o => outOfOrderEvents.Add(o));
var output = new List <PartitionedStreamEvent <int, int> >();
var egress = qc.RegisterOutput(ingress).ForEachAsync(o => output.Add(o));
var process = qc.Restore();
process.Flush();
egress.Wait();
diagnosticEvents = outOfOrderEvents;
dataEvents = output;
}
public FlushOnBatchBoundary_OnCompletedEndOfStream() : base(DisorderPolicy.Throw(), FlushPolicy.FlushOnBatchBoundary, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.EndOfStream)
{
}
public FlushNone_OnCompletedEndOfStream() : base(DisorderPolicy.Throw(), FlushPolicy.None, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.EndOfStream)
{
}
public FlushOnPunctuation_OnCompletedFlush() : base(DisorderPolicy.Throw(), FlushPolicy.FlushOnPunctuation, PeriodicPunctuationPolicy.None(), OnCompletedPolicy.Flush)
{
}
public void LowWatermarkGenerationWithReorderLatency()
{
SetupQuery(
DisorderPolicy.Drop(reorderLatency: 50),
PartitionedFlushPolicy.None,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.Time(generationPeriod: 100, lowWatermarkTimestampLag: 10));
// This establishes three partitions with keys 0,1,2
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 0, 2));
// This will not yet generate a low watermark, since 99 - 0 is less than the generation period of 100
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 99, 0));
// This will generate a low watermark, since (110 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (in this case, none) + generation period (100).
// The timestamp will be 110 - (lowWatermarkTimestampLag of 10) = 100.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 110, 0));
// These will be dropped due to our PartitionedDisorderPolicy.Drop, since the low watermark is now 100,
// even as they approach the low watermark, since any event before the low watermark is out of order.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 99, 1));
// These will be honored, since they are >= low watermark
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 100, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 105, 2));
// This will be reordered since it is >= low watermark and within the partition's
// high watermark (110) - reorderLatency (50)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 100, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 120, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 150, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 190, 2));
// This will generate a low watermark, since (260 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (100) + generation period (100).
// The timestamp will be 260 - (lowWatermarkTimestampLag of 10) = 250.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 260, 0));
// An explicitly ingressed low watermark will update/reset the generationPeriod
this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(300));
// Even though (370 - lowWatermarkTimestampLag of 10) is greater than the last automatically generated
// low watermark timestamp (100) + the generation period (100)); the lowWatermarkGeneration period was
// updated to start at 300, so the next generated low watermark will be at timestamp >=
// (300 + generationPeriod of 100)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 370, 1));
// This will generate a low watermark, since (410 - lowWatermarkTimestampLag of 10) is >= the last low
// watermark timestamp (300) + the generation period (100).
// The timestamp will be 410 - (lowWatermarkTimestampLag of 10) = 400.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 410, 1));
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(2, 0, 2),
PartitionedStreamEvent.CreatePoint(0, 99, 0),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:100)
PartitionedStreamEvent.CreateLowWatermark <int, int>(100),
PartitionedStreamEvent.CreatePoint(0, 100, 1),
PartitionedStreamEvent.CreatePoint(2, 100, 0),
PartitionedStreamEvent.CreatePoint(1, 105, 2),
PartitionedStreamEvent.CreatePoint(2, 110, 0),
PartitionedStreamEvent.CreatePoint(0, 120, 0),
PartitionedStreamEvent.CreatePoint(1, 150, 1),
PartitionedStreamEvent.CreatePoint(2, 190, 2),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:260),
// snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreateLowWatermark <int, int>(200),
PartitionedStreamEvent.CreatePoint(2, 260, 0),
// Explicitly ingressed low watermark
PartitionedStreamEvent.CreateLowWatermark <int, int>(300),
PartitionedStreamEvent.CreatePoint(0, 370, 1),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 0:410)
PartitionedStreamEvent.CreateLowWatermark <int, int>(400),
PartitionedStreamEvent.CreatePoint(0, 410, 1),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
