private void AddEvent(int key, long time)
{
var dataEvent = PartitionedStreamEvent.CreateInterval(key, time, time + IntervalLength, key);
this.input.Add(dataEvent);
UpdateExpectedLowWatermark(time);
long localMin = Math.Max(this.partitionHighWatermark[key] - this.ReorderLatency, this.lowWatermark);
bool locallyOrdered = time >= localMin;
long batchBorder = localMin;
if (locallyOrdered)
{
InsertSorted(key, dataEvent, (time == batchBorder) || this.ReorderLatency == 0);
}
else if (this.disorderPolicy.type == DisorderPolicyType.Drop || batchBorder >= dataEvent.EndTime)
{
AddDiagnostic(key, time); // Drop
}
else // Adjust
{
AddDiagnostic(key, time, batchBorder - dataEvent.SyncTime);
var adjustedEvent = PartitionedStreamEvent.CreateInterval(key, batchBorder, dataEvent.EndTime, key);
InsertSorted(key, adjustedEvent, true); // Since we adjust to batchBorder, batch immediately
}
this.highWatermark = Math.Max(this.highWatermark, time);
this.partitionHighWatermark[key] = Math.Max(this.partitionHighWatermark[key], time);
UpdateBatchMarker(key, batchBorder);
}
public void DisjointUnionNegativeWatermarkRepro()
{
const int leftKey = 1;
const int rightKey = 2;
var left = new Subject <PartitionedStreamEvent <int, int> >();
var right = new Subject <PartitionedStreamEvent <int, int> >();
var qc = new QueryContainer();
var leftInput = qc.RegisterInput(left);
var rightInput = qc.RegisterInput(right);
var actualOutput = new List <PartitionedStreamEvent <int, int> >();
var inputs = new IStreamable <PartitionKey <int>, int>[] { leftInput, rightInput };
var union = new MultiUnionStreamable <PartitionKey <int>, int>(inputs, guaranteedDisjoint: true);
var egress = qc.RegisterOutput(union).ForEachAsync(o => actualOutput.Add(o));
var process = qc.Restore();
left.OnNext(PartitionedStreamEvent.CreatePoint(leftKey, 100, 1));
right.OnNext(PartitionedStreamEvent.CreatePoint(rightKey, 100, 1));
process.Flush();
left.OnCompleted();
right.OnCompleted();
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreatePoint(leftKey, 100, 1),
PartitionedStreamEvent.CreatePoint(rightKey, 100, 1),
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(actualOutput));
}
private void RunAndValidate()
{
// Flush
AddEvent(0, 200);
AddEvent(1, 200);
AddEvent(2, 200);
ProcessInput(out var diagnosticEvents, out var dataEvents);
for (int key = 0; key < 3; key++)
{
var keyData = dataEvents.Where(@event => @event.IsData && @event.PartitionKey == key).ToList();
var expectedData = this.expected[key];
Assert.IsTrue(expectedData.SequenceEqual(keyData));
// Don't bother validating order of diagnostics
var keyDiagnostic = diagnosticEvents.Where(@event => @event.Event.PartitionKey == key)
.OrderBy(e => e.Event.SyncTime).ThenBy(e => e.Event.EndTime).ToList();
var expectedDiagnostic = this.diagnostic[key].OrderBy(e => e.Event.SyncTime).ThenBy(e => e.Event.EndTime).ToList();
Assert.IsTrue(expectedDiagnostic.SequenceEqual(keyDiagnostic));
}
var lowWatermarks = dataEvents.Where(@event => @event.IsLowWatermark).ToList();
this.expectedLowWatermarks.Add(PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime));
Assert.IsTrue(this.expectedLowWatermarks.SequenceEqual(lowWatermarks));
}
public void TestSetup(DisorderPolicy disorderPolicy, PeriodicLowWatermarkPolicy lowWatermarkPolicy)
{
this.disorderPolicy = disorderPolicy;
this.lowWatermarkPolicy = lowWatermarkPolicy;
// This establishes three partitions with keys 0,1,2
PartitionedStreamEvent <int, int> point;
for (int key = 0; key < 3; key++)
{
point = PartitionedStreamEvent.CreatePoint(key, 0, key);
this.input.Add(point);
this.expected[key].Add(point);
this.batchMarker[key] = 1;
}
// Add a point at 100 for key 0 as a baseline, and a low watermark if necessary
this.highWatermark = 100;
point = PartitionedStreamEvent.CreatePoint(0, this.highWatermark, 0);
this.input.Add(point);
UpdateExpectedLowWatermark(this.highWatermark);
this.expected[0].Insert(this.batchMarker[0]++, point);
// If we have a nonzero reorder latency, the event at 100 will be buffered, so add another point at
// 100+ReorderLatency to push the real highwatermark to 100
if (this.ReorderLatency > 0)
{
AddEvent(0, this.highWatermark + this.ReorderLatency);
}
}
private void TumblingWindowWorker <TPayload>(
Func <IPartitionedIngressStreamable <int, TPayload>, IStreamable <PartitionKey <int>, double> > createQuery,
Func <int, long, TPayload> resultCreator)
{
SetupQuery(createQuery, out var input);
input.OnNext(CreateInputInterval(key: 1, startTime: 120, resultCreator));
input.OnNext(CreateInputInterval(key: 2, startTime: 120, resultCreator));
input.OnNext(CreateInputInterval(key: 1, startTime: 150, resultCreator));
input.OnNext(CreateInputInterval(key: 1, startTime: 250, resultCreator));
input.OnNext(CreateInputInterval(key: 2, startTime: 250, resultCreator));
input.OnCompleted();
var expected = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateLowWatermark <int, double>(100),
PartitionedStreamEvent.CreateLowWatermark <int, double>(200),
CreateOutputInterval(key: 1, startTime: 200, endTime: 300, average: 135),
CreateOutputInterval(key: 2, startTime: 200, endTime: 300, average: 120),
PartitionedStreamEvent.CreateLowWatermark <int, double>(300),
CreateOutputInterval(key: 1, startTime: 300, endTime: 400, average: 250),
CreateOutputInterval(key: 2, startTime: 300, endTime: 400, average: 250),
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
FinishQuery(input, expected);
}
public void PassThroughOutOfOrderWithPunctuation1()
{
var input = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0),
PartitionedStreamEvent.CreateStart(2, 2, 2),
PartitionedStreamEvent.CreateStart(1, 1, 1),
PartitionedStreamEvent.CreateStart(3, 3, 3),
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 1),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(2, 3),
PartitionedStreamEvent.CreatePunctuation <int, int>(3, 4),
};
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0),
PartitionedStreamEvent.CreateStart(2, 2, 2),
PartitionedStreamEvent.CreateStart(1, 1, 1),
PartitionedStreamEvent.CreateStart(3, 3, 3),
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 1),
PartitionedStreamEvent.CreatePunctuation <int, int>(1, 2),
PartitionedStreamEvent.CreatePunctuation <int, int>(2, 3),
PartitionedStreamEvent.CreatePunctuation <int, int>(3, 4),
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
var output = Passthrough(input).ToList();
Assert.IsTrue(expected.SequenceEqual(output));
}
public void FlushOnLowWatermarkDualStream()
{
SetupQuery(DisorderPolicy.Throw(reorderLatency: 500));
// These will be buffered due to reorderLatency
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 1, 0)); // key 0: 1-2
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 101, 0)); // key 0: 101-102
// This will batch the first point (0: 1-2)); but the second point ( key 0: 101-102) remains batched, since 101 > 100.
// Since we use the default policy, FlushOnLowWatermark, the first point will also be flushed at this point.
this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(100));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 1005, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 1100, 0));
// OnCompletedPolicy.EndOfStream adds the last two points to the batch then flushes all
var expected = new PartitionedStreamEvent <int, int>[]
{
// Flushed in response to low watermark
PartitionedStreamEvent.CreatePoint(0, 1, 0),
PartitionedStreamEvent.CreateLowWatermark <int, int>(100),
// Flushed in response to OnCompleted
PartitionedStreamEvent.CreatePoint(0, 101, 0),
PartitionedStreamEvent.CreatePoint(0, 1005, 0),
PartitionedStreamEvent.CreatePoint(0, 1100, 0),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public void SumSessionWindow()
{
var input = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 1.0),
PartitionedStreamEvent.CreatePoint(0, 1, 2.0),
PartitionedStreamEvent.CreatePoint(0, 2, 3.0),
PartitionedStreamEvent.CreatePoint(0, 5, 4.0),
PartitionedStreamEvent.CreatePoint(0, 6, 5.0),
PartitionedStreamEvent.CreatePoint(0, 9, 6.0),
};
var expected = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 1.0),
PartitionedStreamEvent.CreateEnd(0, 1, 0, 1.0),
PartitionedStreamEvent.CreateStart(0, 1, 3.0),
PartitionedStreamEvent.CreateEnd(0, 2, 1, 3.0),
PartitionedStreamEvent.CreateStart(0, 2, 6.0),
PartitionedStreamEvent.CreateEnd(0, 4, 2, 6.0),
PartitionedStreamEvent.CreateStart(0, 5, 4.0),
PartitionedStreamEvent.CreateEnd(0, 6, 5, 4.0),
PartitionedStreamEvent.CreateStart(0, 6, 9.0),
PartitionedStreamEvent.CreateEnd(0, 8, 6, 9.0),
PartitionedStreamEvent.CreateStart(0, 9, 6.0),
PartitionedStreamEvent.CreateEnd(0, 11, 9, 6.0),
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
var output = SessionWindowSum(input, 2, 5).ToArray();
Assert.IsTrue(expected.SequenceEqual(output));
}
private static void PartitionedStartEdgeJoinTest()
{
var input1 = new[]
{
PartitionedStreamEvent.CreateStart("Partition1", 100, "A1"),
PartitionedStreamEvent.CreateStart("Partition1", 101, "B1"),
PartitionedStreamEvent.CreateStart("Partition1", 102, "C1"),
PartitionedStreamEvent.CreateStart("Partition2", 100, "A1"),
PartitionedStreamEvent.CreateStart("Partition2", 101, "B1"),
PartitionedStreamEvent.CreateStart("Partition2", 102, "C1"),
};
var input2 = new[]
{
PartitionedStreamEvent.CreateStart("Partition1", 105, "A2"),
PartitionedStreamEvent.CreateStart("Partition1", 106, "B2"),
PartitionedStreamEvent.CreateStart("Partition1", 107, "D2"),
PartitionedStreamEvent.CreateStart("Partition2", 108, "A2"),
PartitionedStreamEvent.CreateStart("Partition2", 109, "D2"),
PartitionedStreamEvent.CreateStart("Partition2", 110, "C2"),
};
// Set properties to start-edge only
var inputStream1 = input1.ToObservable().ToStreamable();
inputStream1.Properties.IsConstantDuration = true;
inputStream1.Properties.ConstantDurationLength = StreamEvent.InfinitySyncTime;
var inputStream2 = input2.ToObservable().ToStreamable();
inputStream2.Properties.IsConstantDuration = true;
inputStream2.Properties.ConstantDurationLength = StreamEvent.InfinitySyncTime;
var output = new List <PartitionedStreamEvent <string, string> >();
inputStream1
.Join(
inputStream2,
l => (l != null ? l[0].ToString() : null),
r => (r != null ? r[0].ToString() : null),
(l, r) => l + "," + r)
.ToStreamEventObservable()
.ForEachAsync(e => output.Add(e))
.Wait();
var correct = new[]
{
PartitionedStreamEvent.CreateStart("Partition1", 105, "A1,A2"),
PartitionedStreamEvent.CreateStart("Partition1", 106, "B1,B2"),
PartitionedStreamEvent.CreateStart("Partition2", 108, "A1,A2"),
PartitionedStreamEvent.CreateStart("Partition2", 110, "C1,C2"),
PartitionedStreamEvent.CreateLowWatermark <string, string>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.SequenceEqual(correct));
}
public void AverageSessionWindow()
{
var input = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0.0),
PartitionedStreamEvent.CreateStart(0, 1, 1.0),
PartitionedStreamEvent.CreateStart(0, 2, 2.0),
PartitionedStreamEvent.CreateStart(0, 8, 4.0),
PartitionedStreamEvent.CreateStart(0, 9, 7.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 8, 2.0),
PartitionedStreamEvent.CreateStart(2, 20, 3.0),
PartitionedStreamEvent.CreateStart(2, 22, 4.0),
PartitionedStreamEvent.CreateStart(3, 3, 3.0),
PartitionedStreamEvent.CreateStart(3, 4, 4.0),
PartitionedStreamEvent.CreateStart(3, 5, 5.0),
};
var expected = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0.0),
PartitionedStreamEvent.CreateEnd(0, 1, 0, 0.0),
PartitionedStreamEvent.CreateStart(0, 1, 0.5),
PartitionedStreamEvent.CreateEnd(0, 2, 1, 0.5),
PartitionedStreamEvent.CreateStart(0, 2, 1.0),
PartitionedStreamEvent.CreateEnd(0, 7, 2, 1.0),
PartitionedStreamEvent.CreateStart(0, 8, 4.0),
PartitionedStreamEvent.CreateEnd(0, 9, 8, 4.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateEnd(1, 2, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 1.5),
PartitionedStreamEvent.CreateEnd(1, 3, 2, 1.5),
PartitionedStreamEvent.CreateStart(2, 8, 2.0),
PartitionedStreamEvent.CreateEnd(2, 13, 8, 2.0),
PartitionedStreamEvent.CreateStart(2, 20, 3.0),
PartitionedStreamEvent.CreateEnd(2, 22, 20, 3.0),
PartitionedStreamEvent.CreateStart(3, 3, 3.0),
PartitionedStreamEvent.CreateEnd(3, 4, 3, 3.0),
PartitionedStreamEvent.CreateStart(3, 4, 3.5),
PartitionedStreamEvent.CreateEnd(3, 5, 4, 3.5),
PartitionedStreamEvent.CreateStart(0, 9, 5.5),
PartitionedStreamEvent.CreateEnd(0, 14, 9, 5.5),
PartitionedStreamEvent.CreateStart(1, 3, 2.0),
PartitionedStreamEvent.CreateEnd(1, 8, 3, 2.0),
PartitionedStreamEvent.CreateStart(2, 22, 3.5),
PartitionedStreamEvent.CreateEnd(2, 27, 22, 3.5),
PartitionedStreamEvent.CreateStart(3, 5, 4.0),
PartitionedStreamEvent.CreateEnd(3, 10, 5, 4.0),
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
var output = SessionWindowAverage(input, 5, 10);
Assert.IsTrue(expected.SequenceEqual(output));
}
public void Union0()
{
var input1 = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0.0),
PartitionedStreamEvent.CreateStart(0, 1, 1.0),
PartitionedStreamEvent.CreateStart(0, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 2, 2.0),
PartitionedStreamEvent.CreateStart(2, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 4, 4.0),
};
var input2 = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(2, 0, 0.0),
PartitionedStreamEvent.CreateStart(2, 1, 1.0),
PartitionedStreamEvent.CreateStart(2, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 3, 3.0),
PartitionedStreamEvent.CreateStart(0, 2, 2.0),
PartitionedStreamEvent.CreateStart(0, 3, 3.0),
PartitionedStreamEvent.CreateStart(0, 4, 4.0),
};
var output = Union(input1, input2).ToArray();
var expected = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0.0),
PartitionedStreamEvent.CreateStart(2, 0, 0.0),
PartitionedStreamEvent.CreateStart(2, 1, 1.0),
PartitionedStreamEvent.CreateStart(2, 2, 2.0),
PartitionedStreamEvent.CreateStart(2, 2, 2.0),
PartitionedStreamEvent.CreateStart(2, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 4, 4.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 3, 3.0),
PartitionedStreamEvent.CreateStart(1, 3, 3.0),
PartitionedStreamEvent.CreateStart(0, 1, 1.0),
PartitionedStreamEvent.CreateStart(0, 2, 2.0),
PartitionedStreamEvent.CreateStart(0, 2, 2.0),
PartitionedStreamEvent.CreateStart(0, 3, 3.0),
PartitionedStreamEvent.CreateStart(0, 4, 4.0),
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(output));
}
private void HoppingWindowWorker <TPayload>(
Func <IPartitionedIngressStreamable <int, TPayload>, IStreamable <PartitionKey <int>, double> > createQuery,
Func <int, long, TPayload> resultCreator)
{
SetupQuery(createQuery, out var input);
// Establish two partitions with keys 1,2
input.OnNext(CreateInputInterval(key: 1, startTime: 100, resultCreator));
input.OnNext(CreateInputInterval(key: 2, startTime: 100, resultCreator));
// Move partition 1 ahead, generating a low watermark at 150 (due to the lifetime alteration of hopping window).
// Partition 2 should not be cleaned up, since it still contains data in its sliding window
input.OnNext(CreateInputInterval(key: 1, startTime: 150, resultCreator));
// Move partition 1 ahead again, generating a low watermark at 250-20=230. Partition 2 can now be cleaned up,
// since it no longer contains data in its sliding window
input.OnNext(CreateInputInterval(key: 1, startTime: 250, resultCreator));
// Reactivate partition 2, which should start with a clean slate
input.OnNext(CreateInputInterval(key: 2, startTime: 250, resultCreator));
input.OnCompleted();
var expected = new PartitionedStreamEvent <int, double>[]
{
// Two partitions established with new hop window starting at 100
PartitionedStreamEvent.CreateLowWatermark <int, double>(100),
CreateOutputStart(key: 1, startTime: 100, average: 100),
CreateOutputStart(key: 2, startTime: 100, average: 100),
// New event for partition 1's sliding window (150), establishes a new hop window starting at 150
PartitionedStreamEvent.CreateLowWatermark <int, double>(150),
CreateOutputEnd(key: 1, endTime: 150, originalStart: 100, average: 100),
CreateOutputStart(key: 1, startTime: 150, average: 125), // Average(100, 150)=125
// New event for partition 1's sliding window (250) creates a low watermark, which deaccumulates
// both events at startTime 100. Partition 2 can be cleaned up (no more data), but partition 1 should
// see its average updated to exclude the 100.
CreateOutputEnd(key: 1, endTime: 200, originalStart: 150, average: 125),
CreateOutputEnd(key: 2, endTime: 200, originalStart: 100, average: 100),
CreateOutputStart(key: 1, startTime: 200, average: 150), // Average(150)=150
CreateOutputEnd(key: 1, endTime: 250, originalStart: 200, average: 150),
PartitionedStreamEvent.CreateLowWatermark <int, double>(250),
// OnCompletedPolicy.EndOfStream - Both partitions accumulate the 250
CreateOutputStart(key: 1, startTime: 250, average: 250), // Average(250)=250
CreateOutputStart(key: 2, startTime: 250, average: 250), // Average(250)=250
CreateOutputEnd(key: 1, endTime: 350, originalStart: 250, average: 250),
CreateOutputEnd(key: 2, endTime: 350, originalStart: 250, average: 250),
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
FinishQuery(input, expected);
}
public void PassThroughEmpty()
{
var input = Array.Empty <PartitionedStreamEvent <int, int> >();
var expected = new PartitionedStreamEvent <int, int>[]
{
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
var output = Passthrough(input).ToList();
Assert.IsTrue(expected.SequenceEqual(output));
}
public void AlterLifeTimeWithFlushDoubles()
{
var startDate = new DateTime(100);
var data = new[]
{
new { Time = startDate, DeviceId = 4, Value = 4 },
new { Time = startDate, DeviceId = 4, Value = 4 },
new { Time = startDate, DeviceId = 6, Value = 6 },
new { Time = startDate, DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(1), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(1), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(2), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(2), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(3), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(3), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(11), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(11), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(5), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(5), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(12), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(12), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(13), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(13), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(14), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(14), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(15), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(15), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(16), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(16), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(17), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(17), DeviceId = 4, Value = 4 },
new { Time = startDate.AddMinutes(21), DeviceId = 6, Value = 6 },
new { Time = startDate.AddMinutes(21), DeviceId = 6, Value = 6 },
}
.Select(e => PartitionedStreamEvent.CreateInterval(e.DeviceId, e.Time.Ticks, StreamEvent.MaxSyncTime, e.Value));
var res = new List <Notification <PartitionedStreamEvent <int, int> > >();
using (data.ToObservable()
.ToStreamable(
DisorderPolicy.Adjust(0),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.Time(36000))
.AlterEventDuration(1)
.ToStreamEventObservable()
.Subscribe(res.NotificationListObserver()))
{
}
Assert.IsTrue(res.Count > 0, "There should be some results.");
}
public void ReorderLatencyWithExplicitPunctuation()
{
var disorderPolicy = DisorderPolicy.Drop(reorderLatency: 10);
SetupQuery(disorderPolicy);
// These will be buffered and reordered due to reorderLatency of 10
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 30, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 25, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 20, 0));
// This will update partition 0's current time to 30. Anything before 30 after this point will be out of order.
this.input.OnNext(PartitionedStreamEvent.CreatePunctuation <int, int>(0, 30));
// Even though these events are before the partitions high watermark (30) - reorder latency (10) = 20,
// they will still be dropped due to the punctuation at 30.
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 25, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 20, 0));
// Events at or after the punctuation are still valid
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 30, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 35, 0));
// Each partition is independent, so a new partition is unaffected by another's high watermark/reorderLatency
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 1, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 10, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 5, 0));
var expected = new PartitionedStreamEvent <int, int>[]
{
// Events are reordered within the reorderLatency
PartitionedStreamEvent.CreatePoint(0, 20, 0),
PartitionedStreamEvent.CreatePoint(0, 25, 0),
PartitionedStreamEvent.CreatePoint(0, 30, 0),
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 30),
PartitionedStreamEvent.CreatePoint(0, 30, 0),
PartitionedStreamEvent.CreatePoint(0, 35, 0),
// Each partition acts independently
PartitionedStreamEvent.CreatePoint(1, 1, 0),
PartitionedStreamEvent.CreatePoint(1, 5, 0),
PartitionedStreamEvent.CreatePoint(1, 10, 0),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public void SorterDequeueUntillRow()
{
var oldSortingTechnique = Config.IngressSortingTechnique;
Config.IngressSortingTechnique = SortingTechnique.ImpatienceSort;
var input = new List <Tuple <int, int> >()
{
Tuple.Create(0, 1),
Tuple.Create(1, 1),
Tuple.Create(0, 3),
Tuple.Create(0, 5),
Tuple.Create(1, 3),
Tuple.Create(1, 20),
Tuple.Create(0, 6),
Tuple.Create(0, 7),
Tuple.Create(0, 8),
Tuple.Create(1, 2),
Tuple.Create(0, 2),
Tuple.Create(1, 30),
Tuple.Create(0, 13)
};
var expectedoutput = new List <Tuple <int, int> >()
{
Tuple.Create(1, 1),
Tuple.Create(1, 3),
Tuple.Create(1, 20),
Tuple.Create(0, 1),
Tuple.Create(0, 2),
Tuple.Create(0, 3),
Tuple.Create(0, 5),
Tuple.Create(0, 6),
Tuple.Create(0, 7),
Tuple.Create(0, 8),
Tuple.Create(0, 13),
Tuple.Create(1, 30)
};
var prog = input.Select(x => PartitionedStreamEvent.CreateStart(x.Item1, x.Item2, x.Item2)).ToObservable()
.ToStreamable(DisorderPolicy.Drop(10)).ToStreamEventObservable();
var outevents = prog.ToEnumerable().ToList();
var output = outevents.Where(o => o.IsData).ToList();
var success = output.SequenceEqual(expectedoutput.Select(t => PartitionedStreamEvent.CreateStart(t.Item1, t.Item2, t.Item2)));
Config.IngressSortingTechnique = oldSortingTechnique;
Assert.IsTrue(success);
}
public void Enqueue(ref PartitionedStreamEvent <TKey, TPayload> streamEvent)
{
ImpatienceSorter sorter;
if (!this.sorters.Lookup(streamEvent.PartitionKey, out int index))
{
sorter = new ImpatienceSorter();
this.sorters.Insert(ref index, streamEvent.PartitionKey, sorter);
}
else
{
sorter = this.sorters.entries[index].value;
}
sorter.Enqueue(ref streamEvent);
}
public void AverageGeneral()
{
var input = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0.0),
PartitionedStreamEvent.CreateStart(0, 1, 1.0),
PartitionedStreamEvent.CreateStart(0, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 2, 2.0),
PartitionedStreamEvent.CreateStart(2, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 4, 4.0),
PartitionedStreamEvent.CreateStart(3, 3, 3.0),
PartitionedStreamEvent.CreateStart(3, 4, 4.0),
PartitionedStreamEvent.CreateStart(3, 5, 5.0),
};
var expected = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0.0),
PartitionedStreamEvent.CreateEnd(0, 1, 0, 0.0),
PartitionedStreamEvent.CreateStart(0, 1, 0.5),
PartitionedStreamEvent.CreateEnd(0, 2, 1, 0.5),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateEnd(1, 2, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 1.5),
PartitionedStreamEvent.CreateEnd(1, 3, 2, 1.5),
PartitionedStreamEvent.CreateStart(2, 2, 2.0),
PartitionedStreamEvent.CreateEnd(2, 3, 2, 2.0),
PartitionedStreamEvent.CreateStart(2, 3, 2.5),
PartitionedStreamEvent.CreateEnd(2, 4, 3, 2.5),
PartitionedStreamEvent.CreateStart(3, 3, 3.0),
PartitionedStreamEvent.CreateEnd(3, 4, 3, 3.0),
PartitionedStreamEvent.CreateStart(3, 4, 3.5),
PartitionedStreamEvent.CreateEnd(3, 5, 4, 3.5),
PartitionedStreamEvent.CreateStart(0, 2, 1.0),
PartitionedStreamEvent.CreateStart(1, 3, 2.0),
PartitionedStreamEvent.CreateStart(2, 4, 3.0),
PartitionedStreamEvent.CreateStart(3, 5, 4.0),
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
var output = AverageGeneral(input).ToList();
Assert.IsTrue(expected.SequenceEqual(output));
}
private void InsertSorted(int key, PartitionedStreamEvent <int, int> dataEvent, bool batchImmediately)
{
if (batchImmediately)
{
this.expected[key].Insert(this.batchMarker[key]++, dataEvent);
}
else
{
int index = this.batchMarker[key];
while (index < this.expected[key].Count && this.expected[key][index].SyncTime < dataEvent.SyncTime)
{
index++;
}
this.expected[key].Insert(index, dataEvent);
}
}
public void Enqueue(ref PartitionedStreamEvent <TKey, TPayload> streamEvent)
{
int loc;
if (streamEvent.SyncTime >= this.Tails[0])
{
loc = 0;
}
else
{
loc = BinarySearch(streamEvent.SyncTime);
}
// Add a new queue
if (loc == this.NumFibers)
{
// Double space to support more queues
if (this.NumFibers >= this.MaxFibers)
{
long[] tmp = new long[this.MaxFibers * 2];
Array.Copy(this.Tails, tmp, this.MaxFibers);
this.Tails = tmp;
this.MaxFibers = this.MaxFibers * 2;
this.MergeSource = new PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > [this.MaxFibers];
}
// Add new queue
this.ecbPool.Get(out PooledElasticCircularBuffer <PartitionedStreamEvent <TKey, TPayload> > ecb);
this.Fibers.Add(ecb);
this.NumFibers++;
}
this.Fibers[loc].Enqueue(ref streamEvent);
if ((loc > 0) && (this.Fibers[loc].Count == 1))
{
if (streamEvent.SyncTime < this.NextAffectingSyncTime)
{
this.NextAffectingSyncTime = streamEvent.SyncTime;
}
}
this.Tails[loc] = streamEvent.SyncTime;
}
public void ReorderLatencyAdjust()
{
var disorderPolicy = DisorderPolicy.Adjust(reorderLatency: 10);
SetupQuery(disorderPolicy);
// These will be buffered and reordered due to reorderLatency of 10
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 30, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 25, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 20, 0));
// These will be adjusted to the minimum, since they are before the partition's high watermark (30) - reorder latency (10) = 20
this.input.OnNext(PartitionedStreamEvent.CreateInterval(0, 8, 30, 0)); // Will be adjusted to 20
this.input.OnNext(PartitionedStreamEvent.CreateInterval(0, 9, 30, 0)); // Will be adjusted to 20
// This will be dropped, since Trill will not adjust intervals whose end times are before the minimum
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 8, 0)); // Will be dropped
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 9, 0)); // Will be dropped
// Each partition is independent, so a new partition is unaffected by another's high watermark/reorderLatency
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 1, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 10, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 5, 0));
var expected = new PartitionedStreamEvent <int, int>[]
{
// Events are reordered within the reorderLatency, and out of order events are adjusted to minimum
PartitionedStreamEvent.CreatePoint(0, 20, 0),
PartitionedStreamEvent.CreateInterval(0, 20, 30, 0),
PartitionedStreamEvent.CreateInterval(0, 20, 30, 0),
PartitionedStreamEvent.CreatePoint(0, 25, 0),
PartitionedStreamEvent.CreatePoint(0, 30, 0),
// Each partition acts independently
PartitionedStreamEvent.CreatePoint(1, 1, 0),
PartitionedStreamEvent.CreatePoint(1, 5, 0),
PartitionedStreamEvent.CreatePoint(1, 10, 0),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public IEnumerable <index_t> Parse()
{
using var reader = new StreamReader(filename);
var index = new List <index_t>();
while (!reader.EndOfStream)
{
var line = reader.ReadLine();
if (line != null)
{
var values = line.Split(',');
var devid = values[1].Trim();
var s = long.Parse(values[2]);
var e = long.Parse(values[3]);
index.Add(PartitionedStreamEvent.CreateInterval(devid, s, e, Unit.Default));
}
}
return(index);
}
private void UpdateExpectedLowWatermark(long eventTime)
{
if (this.GenerateLowWatermarks && eventTime > this.Lag)
{
var newLowWatermarkTime = eventTime - this.Lag;
if (newLowWatermarkTime >= this.lowWatermark + this.GenerationPeriod && newLowWatermarkTime > this.lowWatermark)
{
// eventTime is sufficiently high to generate a new watermark, but first snap it to the nearest generationPeriod boundary
newLowWatermarkTime = newLowWatermarkTime.SnapToLeftBoundary((long)this.lowWatermarkPolicy.generationPeriod);
this.lowWatermark = newLowWatermarkTime;
for (int key = 0; key < 3; key++)
{
this.partitionHighWatermark[key] = Math.Max(this.partitionHighWatermark[key], newLowWatermarkTime);
UpdateBatchMarker(key, newLowWatermarkTime);
}
this.expectedLowWatermarks.Add(PartitionedStreamEvent.CreateLowWatermark <int, int>(newLowWatermarkTime));
}
}
}
public void FlushOnLowWatermarkSimple()
{
SetupQuery(DisorderPolicy.Throw(reorderLatency: 10), PartitionedFlushPolicy.FlushOnLowWatermark,
null, null, OnCompletedPolicy.None);
// This will be buffered due to reorderLatency
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 1, 0));
// Since we specify FlushOnLowWatermark, this will flush the point above, as well as the low watermark itself
this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(9000));
var expected = new PartitionedStreamEvent <int, int>[]
{
// Flushed in response to low watermark
PartitionedStreamEvent.CreatePoint(0, 1, 0),
PartitionedStreamEvent.CreateLowWatermark <int, int>(9000),
};
FinishQuery(expected);
}
public void LowWatermark()
{
var disorderPolicy = DisorderPolicy.Drop(reorderLatency: 10);
SetupQuery(disorderPolicy, PartitionedFlushPolicy.None);
// This will be buffered due to reorderLatency
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 1, 0));
// This will set the entire stream's lower bound to 100
this.input.OnNext(PartitionedStreamEvent.CreateLowWatermark <int, int>(100));
// If any partition, new or existing, specifies a timestamp before 100, it is considered out of order
// and since we specify DisorderPolicy.Drop, they will be dropped
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 50, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 50, 0));
// Any points at or after the low watermark of 100 will be processed as usual
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 100, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 100, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 105, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 105, 0));
var expected = new PartitionedStreamEvent <int, int>[]
{
// Flushed in response to the default OnCompletedPolicy.EndOfStream
PartitionedStreamEvent.CreatePoint(0, 1, 0),
PartitionedStreamEvent.CreateLowWatermark <int, int>(100),
// Points at time 50 are dropped
PartitionedStreamEvent.CreatePoint(0, 100, 0),
PartitionedStreamEvent.CreatePoint(1, 100, 0),
PartitionedStreamEvent.CreatePoint(0, 105, 0),
PartitionedStreamEvent.CreatePoint(1, 105, 0),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public void PunctuationTimeGeneration()
{
SetupQuery(
DisorderPolicy.Throw(),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.Time(generationPeriod: 1000));
// This establishes three partitions with keys 1,2,3
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 0, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 0, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 0, 2));
// Since the next point arrives at 10001, which is exactly PeriodicPunctuationPolicy.generationPeriod after
// the first point, this will generate a punctuation at 10001
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 10001, 0));
// These are still valid, since the punctuation generated above only affects key 0
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 1, 1));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(2, 1, 2));
var expected = new PartitionedStreamEvent <int, int>[]
{
// Flushed in response to OnCompleted
PartitionedStreamEvent.CreatePoint(0, 0, 0),
PartitionedStreamEvent.CreatePoint(1, 0, 1),
PartitionedStreamEvent.CreatePoint(2, 0, 2),
// Punctuation snapped to the leftmost generationPeriod boundary
PartitionedStreamEvent.CreatePunctuation <int, int>(0, 10000),
PartitionedStreamEvent.CreatePoint(0, 10001, 0),
PartitionedStreamEvent.CreatePoint(1, 1, 1),
PartitionedStreamEvent.CreatePoint(2, 1, 2),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
public void AverageTumblingWindow()
{
var input = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateStart(0, 0, 0.0),
PartitionedStreamEvent.CreateStart(0, 1, 1.0),
PartitionedStreamEvent.CreateStart(0, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 1, 1.0),
PartitionedStreamEvent.CreateStart(1, 2, 2.0),
PartitionedStreamEvent.CreateStart(1, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 2, 2.0),
PartitionedStreamEvent.CreateStart(2, 3, 3.0),
PartitionedStreamEvent.CreateStart(2, 4, 4.0),
PartitionedStreamEvent.CreateStart(3, 3, 3.0),
PartitionedStreamEvent.CreateStart(3, 4, 4.0),
PartitionedStreamEvent.CreateStart(3, 5, 5.0),
};
var expected = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateInterval(0, 0, 1, 0.0),
PartitionedStreamEvent.CreateInterval(0, 1, 2, 1.0),
PartitionedStreamEvent.CreateInterval(1, 1, 2, 1.0),
PartitionedStreamEvent.CreateInterval(1, 2, 3, 2.0),
PartitionedStreamEvent.CreateInterval(2, 2, 3, 2.0),
PartitionedStreamEvent.CreateInterval(2, 3, 4, 3.0),
PartitionedStreamEvent.CreateInterval(3, 3, 4, 3.0),
PartitionedStreamEvent.CreateInterval(3, 4, 5, 4.0),
PartitionedStreamEvent.CreateInterval(0, 2, 3, 2.0),
PartitionedStreamEvent.CreateInterval(1, 3, 4, 3.0),
PartitionedStreamEvent.CreateInterval(2, 4, 5, 4.0),
PartitionedStreamEvent.CreateInterval(3, 5, 6, 5.0),
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
var output = TumblingWindow(input);
Assert.IsTrue(expected.SequenceEqual(output));
}
public void ReorderLatencyDrop()
{
var disorderPolicy = DisorderPolicy.Drop(reorderLatency: 10);
SetupQuery(disorderPolicy);
// These will be buffered and reordered due to reorderLatency of 10
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 30, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 25, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 20, 0));
// These will be dropped since they are before the partition's high watermark (30) - reorder latency (10) = 20
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 8, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(0, 9, 0));
// Each partition is independent, so a new partition is unaffected by another's high watermark/reorderLatency
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 1, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 10, 0));
this.input.OnNext(PartitionedStreamEvent.CreatePoint(1, 5, 0));
var expected = new PartitionedStreamEvent <int, int>[]
{
// Events are reordered within the reorderLatency
PartitionedStreamEvent.CreatePoint(0, 20, 0),
PartitionedStreamEvent.CreatePoint(0, 25, 0),
PartitionedStreamEvent.CreatePoint(0, 30, 0),
// Each partition acts independently
PartitionedStreamEvent.CreatePoint(1, 1, 0),
PartitionedStreamEvent.CreatePoint(1, 5, 0),
PartitionedStreamEvent.CreatePoint(1, 10, 0),
// Generated by the default OnCompletedPolicy EndOfStream
PartitionedStreamEvent.CreateLowWatermark <int, int>(StreamEvent.InfinitySyncTime),
};
FinishQuery(expected);
}
// StartEdge-only streamables should never clean up partitions, since start edges are always valid state
private void StartEdgeWorker <TPayload>(
Func <IPartitionedIngressStreamable <int, TPayload>, IStreamable <PartitionKey <int>, double> > createQuery,
Func <int, long, TPayload> resultCreator)
{
SetupQuery(createQuery, out var input);
input.OnNext(CreateInputInterval(key: 1, startTime: 100, resultCreator));
input.OnNext(CreateInputInterval(key: 2, startTime: 100, resultCreator));
input.OnNext(CreateInputInterval(key: 1, startTime: 160, resultCreator));
input.OnNext(CreateInputInterval(key: 1, startTime: 250, resultCreator));
input.OnNext(CreateInputInterval(key: 2, startTime: 250, resultCreator));
input.OnCompleted();
var expected = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreateLowWatermark <int, double>(80),
CreateOutputStart(key: 1, startTime: 100, average: 100),
CreateOutputStart(key: 2, startTime: 100, average: 100),
PartitionedStreamEvent.CreateLowWatermark <int, double>(140),
CreateOutputEnd(key: 1, endTime: 160, originalStart: 100, average: 100),
CreateOutputStart(key: 1, startTime: 160, average: 130), // Average(100, 160)=130
PartitionedStreamEvent.CreateLowWatermark <int, double>(230),
CreateOutputEnd(key: 1, endTime: 250, originalStart: 160, average: 130),
CreateOutputEnd(key: 2, endTime: 250, originalStart: 100, average: 100),
CreateOutputStart(key: 1, startTime: 250, average: 170), // Average(100, 160, 250)=170
CreateOutputStart(key: 2, startTime: 250, average: 175), // Average(100, 250)=175
PartitionedStreamEvent.CreateLowWatermark <int, double>(StreamEvent.InfinitySyncTime),
};
FinishQuery(input, expected);
}
public void SelectManyWithFlush()
{
var data = new PartitionedStreamEvent <int, double>[]
{
PartitionedStreamEvent.CreatePoint(0, 0, 0.0),
PartitionedStreamEvent.CreatePoint(0, 1, 1.0),
PartitionedStreamEvent.CreatePoint(0, 2, 2.0),
PartitionedStreamEvent.CreatePoint(1, 1, 1.0),
PartitionedStreamEvent.CreatePoint(1, 2, 2.0),
PartitionedStreamEvent.CreatePoint(1, 3, 3.0),
PartitionedStreamEvent.CreatePoint(2, 2, 2.0),
PartitionedStreamEvent.CreatePoint(2, 3, 3.0),
PartitionedStreamEvent.CreatePoint(2, 4, 4.0),
};
var j = new Subject <PartitionedStreamEvent <int, double> >();
var res = new List <Notification <PartitionedStreamEvent <int, double> > >();
var qc = new QueryContainer();
var input = qc.RegisterInput(j);
var output = qc.RegisterOutput(input.SelectMany(e => new[] { e, 100.0 * e }));
using (j)
using (output.Subscribe(res.NotificationListObserver()))
{
Process p = qc.Restore(null);
foreach (var x in data)
{
j.OnNext(x);
}
p.Flush();
Assert.IsTrue(res.Count > data.Length, "Flush should push all events out.");
j.OnCompleted();
}
}
