public void LowWatermarkGenerationWithReorderLatency()
{
SetupQuery(
DisorderPolicy.Drop(reorderLatency: 50),
PartitionedFlushPolicy.None,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.Time(generationPeriod: 100, lowWatermarkTimestampLag: 10));
// This establishes three partitions with keys 0,1,2
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 0, 2));
// This will not yet generate a low watermark, since 99 - 0 is less than the generation period of 100
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 99, 0));
// This will generate a low watermark, since (110 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (in this case, none) + generation period (100).
// The timestamp will be 110 - (lowWatermarkTimestampLag of 10) = 100.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 110, 0));
// These will be dropped due to our PartitionedDisorderPolicy.Drop, since the low watermark is now 100,
// even as they approach the low watermark, since any event before the low watermark is out of order.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 99, 1));
// These will be honored, since they are >= low watermark
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 100, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 105, 2));
// This will be reordered since it is >= low watermark and within the partition's
// high watermark (110) - reorderLatency (50)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 100, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 120, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 150, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 190, 2));
// This will generate a low watermark, since (260 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (100) + generation period (100).
// The timestamp will be 260 - (lowWatermarkTimestampLag of 10) = 250.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 260, 0));
// An explicitly ingressed low watermark will update/reset the generationPeriod
this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(300));
// Even though (370 - lowWatermarkTimestampLag of 10) is greater than the last automatically generated
// low watermark timestamp (100) + the generation period (100)); the lowWatermarkGeneration period was
// updated to start at 300, so the next generated low watermark will be at timestamp >=
// (300 + generationPeriod of 100)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 370, 1));
// This will generate a low watermark, since (410 - lowWatermarkTimestampLag of 10) is >= the last low
// watermark timestamp (300) + the generation period (100).
// The timestamp will be 410 - (lowWatermarkTimestampLag of 10) = 400.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 410, 1));
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(2, 0, 2),
PartitionedStreamEvent.CreatePoint(0, 99, 0),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:100)
PartitionedStreamEvent.CreateLowWatermark <int, int>(100),
PartitionedStreamEvent.CreatePoint(0, 100, 1),
PartitionedStreamEvent.CreatePoint(2, 100, 0),
PartitionedStreamEvent.CreatePoint(1, 105, 2),
PartitionedStreamEvent.CreatePoint(2, 110, 0),
PartitionedStreamEvent.CreatePoint(0, 120, 0),
PartitionedStreamEvent.CreatePoint(1, 150, 1),
PartitionedStreamEvent.CreatePoint(2, 190, 2),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:260),
// snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreateLowWatermark <int, int>(200),
PartitionedStreamEvent.CreatePoint(2, 260, 0),
// Explicitly ingressed low watermark
PartitionedStreamEvent.CreateLowWatermark <int, int>(300),
PartitionedStreamEvent.CreatePoint(0, 370, 1),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 0:410)
PartitionedStreamEvent.CreateLowWatermark <int, int>(400),
PartitionedStreamEvent.CreatePoint(0, 410, 1),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public void LowWatermarkAndPunctuationGeneration()
{
SetupQuery(DisorderPolicy.Throw(), PartitionedFlushPolicy.None,
PeriodicPunctuationPolicy.Time(generationPeriod: 20),
PeriodicLowWatermarkPolicy.Time(generationPeriod: 50, lowWatermarkTimestampLag: 0));
// This establishes three partitions with keys 0,1
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
// These points do not generate punctuations or low watermarks since neither period has lapsed.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 15, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 19, 1));
// These points generate punctuations (period of 20) but not low watermarks (period of 50)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 22, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 20, 1));
// This point generates a low watermark, since it is >= the last watermark (none/0) + the low watermark
// generation period of 50
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 50, 0));
// These points do not generate punctuations, even though "punctuations" specifically haven't been generated
// for these partitions since time 22/20, because the low watermark at 50 supersedes punctuations.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 55, 1));
// Punctuation generation continues once the punctuation generation period lapses from the low watermark
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 70, 2));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 71, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 2));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 91, 1));
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(0, 15, 0),
PartitionedStreamEvent.CreatePoint(1, 19, 1),
// Punctuations generated after the punctuation period of 20 from time 0, snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 20),
PartitionedStreamEvent.CreatePoint(0, 22, 0),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 20),
PartitionedStreamEvent.CreatePoint(1, 20, 1),
// Low watermark generated after the low watermark period of 50 from time 0
PartitionedStreamEvent.CreateLowWatermark <int, int>(50),
PartitionedStreamEvent.CreatePoint(0, 50, 0),
PartitionedStreamEvent.CreatePoint(1, 55, 1),
// Punctuations generated after the punctuation period of 20 from time 50 (low watermark time), snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 60),
PartitionedStreamEvent.CreatePoint(0, 70, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 60),
PartitionedStreamEvent.CreatePoint(1, 71, 1),
// Punctuations generated after the punctuation period of 20 from times 60 (last punctuation time), snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 80),
PartitionedStreamEvent.CreatePoint(0, 90, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 80),
PartitionedStreamEvent.CreatePoint(1, 91, 1),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
private void SetupQuery <TInput>(
Func <IPartitionedIngressStreamable <int, TInput>, IStreamable <PartitionKey <int>, double> > createQuery,
out Subject <PartitionedStreamEvent <int, TInput> > input)
{
const uint generationPeriod = 10;
const uint lowWatermarkTimestampLag = 20;
var qc = new QueryContainer();
input = new Subject <PartitionedStreamEvent <int, TInput> >();
var ingress = qc.RegisterInput(input, null, PartitionedFlushPolicy.None, null, PeriodicLowWatermarkPolicy.Time(generationPeriod, lowWatermarkTimestampLag));
var query = createQuery(ingress);
this.output = new List <PartitionedStreamEvent <int, double> >();
this.egress = qc.RegisterOutput(query).ForEachAsync(o => this.output.Add(o));
this.process = qc.Restore();
}
public void ReorderLatencyDisordering() => ReorderLatencyDisorderingBase(DisorderPolicy.Drop(0), PeriodicLowWatermarkPolicy.Time(20, 5));
public void LocalDisordering() => LocalDisorderingBase(DisorderPolicy.Drop(0), PeriodicLowWatermarkPolicy.Time(20, 5));
public void ReorderLatencyDisordering() => ReorderLatencyDisorderingBase(DisorderPolicy.Adjust(10), PeriodicLowWatermarkPolicy.Time(20, 20));
public void LocalDisordering() => LocalDisorderingBase(DisorderPolicy.Adjust(10), PeriodicLowWatermarkPolicy.Time(20, 20));