protected void ReorderLatencyDisorderingBase(DisorderPolicy disorderPolicy, PeriodicLowWatermarkPolicy lowWatermarkPolicy)
{
TestSetup(disorderPolicy, lowWatermarkPolicy);
// Partition 0 - locally disordered
AddEvent(0, 100);
AddEvent(0, 95);
AddEvent(0, 89);
// Partition 1 - globally disordered
AddEvent(1, 80);
AddEvent(1, 85);
AddEvent(1, 90);
AddEvent(1, 95);
AddEvent(1, 100);
// Partition 2 - locally and globally disordered
AddEvent(2, 80);
AddEvent(2, 85);
AddEvent(2, 90);
AddEvent(2, 95);
AddEvent(2, 100);
AddEvent(2, 99);
AddEvent(2, 94);
AddEvent(2, 89);
AddEvent(2, 84);
AddEvent(2, 79);
RunAndValidate();
}
public void TestSetup(DisorderPolicy disorderPolicy, PeriodicLowWatermarkPolicy lowWatermarkPolicy)
{
this.disorderPolicy = disorderPolicy;
this.lowWatermarkPolicy = lowWatermarkPolicy;
// This establishes three partitions with keys 0,1,2
PartitionedStreamEvent <int, int> point;
for (int key = 0; key < 3; key++)
{
point = PartitionedStreamEvent.CreatePoint(key, 0, key);
this.input.Add(point);
this.expected[key].Add(point);
this.batchMarker[key] = 1;
}
// Add a point at 100 for key 0 as a baseline, and a low watermark if necessary
this.highWatermark = 100;
point = PartitionedStreamEvent.CreatePoint(0, this.highWatermark, 0);
this.input.Add(point);
UpdateExpectedLowWatermark(this.highWatermark);
this.expected[0].Insert(this.batchMarker[0]++, point);
// If we have a nonzero reorder latency, the event at 100 will be buffered, so add another point at
// 100+ReorderLatency to push the real highwatermark to 100
if (this.ReorderLatency > 0)
{
AddEvent(0, this.highWatermark + this.ReorderLatency);
}
}
internal TriPartitionedOrderedTestsBase(
ConfigModifier config,
DisorderPolicy disorderPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
PeriodicLowWatermarkPolicy lowWatermarkPolicy) : base(config)
{
this.disorderPolicy = disorderPolicy;
this.punctuationPolicy = punctuationPolicy;
this.lowWatermarkPolicy = lowWatermarkPolicy;
}
private void SetupQuery(
DisorderPolicy disorderPolicy,
PartitionedFlushPolicy flushPolicy = PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy punctuationPolicy = null, // Default: PeriodicPunctuationPolicy.None()
PeriodicLowWatermarkPolicy lowWatermarkPolicy = null, // Default: PeriodicLowWatermarkPolicy.None()
OnCompletedPolicy completedPolicy = OnCompletedPolicy.EndOfStream)
{
var qc = new QueryContainer();
this.input = new Subject <PartitionedStreamEvent <int, int> >();
var ingress = qc.RegisterInput(this.input, disorderPolicy, flushPolicy, punctuationPolicy,
lowWatermarkPolicy, completedPolicy);
this.output = new List <PartitionedStreamEvent <int, int> >();
this.egress = qc.RegisterOutput(ingress).ForEachAsync(o => this.output.Add(o));
this.process = qc.Restore();
this.validateByPartition = disorderPolicy.reorderLatency > 0;
}
protected void LocalDisorderingBase(DisorderPolicy disorderPolicy, PeriodicLowWatermarkPolicy lowWatermarkPolicy)
{
TestSetup(disorderPolicy, lowWatermarkPolicy);
// Add events that are locally disordered
// Partition 0
AddEvent(0, 100);
// Partition 1
AddEvent(1, 100);
AddEvent(1, 95);
AddEvent(1, 89);
// Partition 2
AddEvent(2, 96);
AddEvent(2, 94);
AddEvent(2, 91);
RunAndValidate();
}
public void LowWatermarkAndPunctuationGeneration()
{
SetupQuery(DisorderPolicy.Throw(), PartitionedFlushPolicy.None,
PeriodicPunctuationPolicy.Time(generationPeriod: 20),
PeriodicLowWatermarkPolicy.Time(generationPeriod: 50, lowWatermarkTimestampLag: 0));
// This establishes three partitions with keys 0,1
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
// These points do not generate punctuations or low watermarks since neither period has lapsed.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 15, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 19, 1));
// These points generate punctuations (period of 20) but not low watermarks (period of 50)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 22, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 20, 1));
// This point generates a low watermark, since it is >= the last watermark (none/0) + the low watermark
// generation period of 50
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 50, 0));
// These points do not generate punctuations, even though "punctuations" specifically haven't been generated
// for these partitions since time 22/20, because the low watermark at 50 supersedes punctuations.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 55, 1));
// Punctuation generation continues once the punctuation generation period lapses from the low watermark
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 70, 2));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 71, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 2));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 91, 1));
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(0, 15, 0),
PartitionedStreamEvent.CreatePoint(1, 19, 1),
// Punctuations generated after the punctuation period of 20 from time 0, snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 20),
PartitionedStreamEvent.CreatePoint(0, 22, 0),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 20),
PartitionedStreamEvent.CreatePoint(1, 20, 1),
// Low watermark generated after the low watermark period of 50 from time 0
PartitionedStreamEvent.CreateLowWatermark <int, int>(50),
PartitionedStreamEvent.CreatePoint(0, 50, 0),
PartitionedStreamEvent.CreatePoint(1, 55, 1),
// Punctuations generated after the punctuation period of 20 from time 50 (low watermark time), snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 60),
PartitionedStreamEvent.CreatePoint(0, 70, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 60),
PartitionedStreamEvent.CreatePoint(1, 71, 1),
// Punctuations generated after the punctuation period of 20 from times 60 (last punctuation time), snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 80),
PartitionedStreamEvent.CreatePoint(0, 90, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 80),
PartitionedStreamEvent.CreatePoint(1, 91, 1),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
private void SetupQuery <TInput>(
Func <IPartitionedIngressStreamable <int, TInput>, IStreamable <PartitionKey <int>, double> > createQuery,
out Subject <PartitionedStreamEvent <int, TInput> > input)
{
const uint generationPeriod = 10;
const uint lowWatermarkTimestampLag = 20;
var qc = new QueryContainer();
input = new Subject <PartitionedStreamEvent <int, TInput> >();
var ingress = qc.RegisterInput(input, null, PartitionedFlushPolicy.None, null, PeriodicLowWatermarkPolicy.Time(generationPeriod, lowWatermarkTimestampLag));
var query = createQuery(ingress);
this.output = new List <PartitionedStreamEvent <int, double> >();
this.egress = qc.RegisterOutput(query).ForEachAsync(o => this.output.Add(o));
this.process = qc.Restore();
}
public void LowWatermarkGenerationWithReorderLatency()
{
SetupQuery(
DisorderPolicy.Drop(reorderLatency: 50),
PartitionedFlushPolicy.None,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.Time(generationPeriod: 100, lowWatermarkTimestampLag: 10));
// This establishes three partitions with keys 0,1,2
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 0, 2));
// This will not yet generate a low watermark, since 99 - 0 is less than the generation period of 100
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 99, 0));
// This will generate a low watermark, since (110 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (in this case, none) + generation period (100).
// The timestamp will be 110 - (lowWatermarkTimestampLag of 10) = 100.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 110, 0));
// These will be dropped due to our PartitionedDisorderPolicy.Drop, since the low watermark is now 100,
// even as they approach the low watermark, since any event before the low watermark is out of order.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 90, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 99, 1));
// These will be honored, since they are >= low watermark
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 100, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 105, 2));
// This will be reordered since it is >= low watermark and within the partition's
// high watermark (110) - reorderLatency (50)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 100, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 120, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 150, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 190, 2));
// This will generate a low watermark, since (260 - lowWatermarkTimestampLag of 10) is greater than the
// last watermark timestamp (100) + generation period (100).
// The timestamp will be 260 - (lowWatermarkTimestampLag of 10) = 250.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 260, 0));
// An explicitly ingressed low watermark will update/reset the generationPeriod
this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(300));
// Even though (370 - lowWatermarkTimestampLag of 10) is greater than the last automatically generated
// low watermark timestamp (100) + the generation period (100)); the lowWatermarkGeneration period was
// updated to start at 300, so the next generated low watermark will be at timestamp >=
// (300 + generationPeriod of 100)
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 370, 1));
// This will generate a low watermark, since (410 - lowWatermarkTimestampLag of 10) is >= the last low
// watermark timestamp (300) + the generation period (100).
// The timestamp will be 410 - (lowWatermarkTimestampLag of 10) = 400.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 410, 1));
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(2, 0, 2),
PartitionedStreamEvent.CreatePoint(0, 99, 0),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:100)
PartitionedStreamEvent.CreateLowWatermark <int, int>(100),
PartitionedStreamEvent.CreatePoint(0, 100, 1),
PartitionedStreamEvent.CreatePoint(2, 100, 0),
PartitionedStreamEvent.CreatePoint(1, 105, 2),
PartitionedStreamEvent.CreatePoint(2, 110, 0),
PartitionedStreamEvent.CreatePoint(0, 120, 0),
PartitionedStreamEvent.CreatePoint(1, 150, 1),
PartitionedStreamEvent.CreatePoint(2, 190, 2),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 2:260),
// snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreateLowWatermark <int, int>(200),
PartitionedStreamEvent.CreatePoint(2, 260, 0),
// Explicitly ingressed low watermark
PartitionedStreamEvent.CreateLowWatermark <int, int>(300),
PartitionedStreamEvent.CreatePoint(0, 370, 1),
// Generated automatically because of PeriodicLowWatermarkPolicy in response to point (key 0:410)
PartitionedStreamEvent.CreateLowWatermark <int, int>(400),
PartitionedStreamEvent.CreatePoint(0, 410, 1),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public void LocalDisordering() => LocalDisorderingBase(DisorderPolicy.Drop(0), PeriodicLowWatermarkPolicy.Time(20, 5));
public void ReorderLatencyDisordering() => ReorderLatencyDisorderingBase(DisorderPolicy.Drop(0), PeriodicLowWatermarkPolicy.Time(20, 5));
public void ReorderLatencyDisordering() => ReorderLatencyDisorderingBase(DisorderPolicy.Adjust(10), PeriodicLowWatermarkPolicy.Time(20, 20));
public void LocalDisordering() => LocalDisorderingBase(DisorderPolicy.Adjust(10), PeriodicLowWatermarkPolicy.Time(20, 20));
public void ReorderLatencyDisordering() => ReorderLatencyDisorderingBase(DisorderPolicy.Adjust(5), PeriodicLowWatermarkPolicy.None());
public void LocalDisordering() => LocalDisorderingBase(DisorderPolicy.Adjust(5), PeriodicLowWatermarkPolicy.None());
public void LocalDisordering() => LocalDisorderingBase(DisorderPolicy.Drop(10), PeriodicLowWatermarkPolicy.None());