public void TestAggregateByKey()
{
var input = new[]
{
StreamEvent.CreateStart(0, 100),
StreamEvent.CreateStart(0, 105),
StreamEvent.CreateStart(0, 104),
StreamEvent.CreateStart(0, 200),
StreamEvent.CreateStart(0, 201),
StreamEvent.CreateStart(0, 300),
StreamEvent.CreateStart(0, 302),
StreamEvent.CreateStart(0, 303),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable().SetProperty().IsConstantDuration(true, StreamEvent.InfinitySyncTime);
var output = input
.AggregateByKey(
a => a / 100,
b => b.Count(),
(key, count) => key * 100 + (int)count * 10);
var correct = new[]
{
StreamEvent.CreateStart(0, 130),
StreamEvent.CreateStart(0, 220),
StreamEvent.CreateStart(0, 330),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void FilterTest()
{
// Creates a basic filter for even-timestamped events, punctuations
var inputSubject = new Subject <StreamEvent <int> >();
var qc = new QueryContainer();
var input = qc.RegisterInput(inputSubject, this.disorderPolicy, this.flushPolicy, this.punctuationPolicy, this.completedPolicy);
IStreamable <Empty, int> query = input;
// Add a no-op operator that isn't simply a batch-in-batch-out operator
query = query.ClipEventDuration(IntervalLength);
query = query.Where(this.FilterExpression);
query = query.ClipEventDuration(IntervalLength);
var filtered = qc.RegisterOutput(query).ForEachAsync(o => OnEgress(o));
var process = qc.Restore();
for (int i = 0; i < IngressEventCount; i++)
{
OnIngress(inputSubject, StreamEvent.CreateInterval(i, i + IntervalLength, i));
if (i > 0 && i % PunctuationGenerationPeriod == 0)
{
OnIngress(inputSubject, StreamEvent.CreatePunctuation <int>(i));
}
// Make sure we don't have any pending events we expected to be egressed at this point
Assert.IsTrue(this.expectedOutput.Count == 0);
}
OnCompleted(inputSubject);
}
public void TestAggregateByKeyTupleDecomposition()
{
var input = new[]
{
StreamEvent.CreateStart(0, 100),
StreamEvent.CreateStart(0, 105),
StreamEvent.CreateStart(0, 104),
StreamEvent.CreateStart(0, 200),
StreamEvent.CreateStart(0, 201),
StreamEvent.CreateStart(0, 300),
StreamEvent.CreateStart(0, 302),
StreamEvent.CreateStart(0, 303),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = new List <StreamEvent <(bool, ulong)> >();
input
.GroupAggregate(s => true, w => w.Count(), (g, c) => ValueTuple.Create(g.Key, c))
.ToStreamEventObservable()
.ForEachAsync(e => output.Add(e))
.Wait();
var correct = new[]
{
StreamEvent.CreateStart(0, ValueTuple.Create(true, (ulong)8)),
StreamEvent.CreatePunctuation <ValueTuple <bool, ulong> >(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(correct.SequenceEqual(output));
}
public void TestSumAggregate_Double()
{
/*
* Time: 1 2 3 4 5
* Input: 1 |---------------|
* 2 |-------|
* 4 |---|
* 8 |-------|
* 16 |---|
* Output: 3 15 25 1 0
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 1.0),
StreamEvent.CreateInterval(1, 3, 2.0),
StreamEvent.CreatePunctuation <double>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Sum(v => v);
var correct = new[]
{
StreamEvent.CreateStart(1, 3.0),
StreamEvent.CreateEnd(3, 1, 3.0),
StreamEvent.CreateStart(3, 1.0),
StreamEvent.CreateEnd(5, 3, 1.0),
StreamEvent.CreatePunctuation <double>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
private static void JoinIntervalsTest()
{
var input1 = new[]
{
StreamEvent.CreateInterval(100, 110, "A1"),
StreamEvent.CreateInterval(101, 111, "B1"),
StreamEvent.CreateInterval(102, 112, "C1"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var input2 = new[]
{
StreamEvent.CreateInterval(105, 115, "A2"),
StreamEvent.CreateInterval(106, 116, "B2"),
StreamEvent.CreateInterval(107, 117, "D2"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var inputStream1 = input1.ToCleanStreamable();
var inputStream2 = input2.ToCleanStreamable();
var outputStream = inputStream1.Join(
inputStream2,
l => (l != null ? l[0].ToString() : null),
r => (r != null ? r[0].ToString() : null),
(l, r) => l + "," + r);
var correct = new[]
{
StreamEvent.CreateInterval(105, 110, "A1,A2"),
StreamEvent.CreateInterval(106, 111, "B1,B2"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(outputStream.IsEquivalentTo(correct));
}
public static StreamCache <Empty, T> Sort <T, U>(this IStreamable <Empty, T> stream, Expression <Func <T, U> > sortBySelector)
{
var u_comparer = Utility.CreateCompoundComparer(sortBySelector, ComparerExpression <U> .Default.GetCompareExpr()).Compile();
IStreamable <Empty, T> stream2 = stream;
stream.GetProperties(out StreamProperties <Empty, T> properties);
if (!properties.IsStartEdgeOnly)
{
var secondStream = new StreamEvent <T>[]
{
StreamEvent.CreateStart(StreamEvent.MaxSyncTime, default(T)),
StreamEvent.CreatePunctuation <T>(StreamEvent.InfinitySyncTime),
}.ToObservable().ToStreamable();
stream2 = stream.Join(secondStream, (x, y) => x);
}
var observableCache = stream2.ToPayloadEnumerable().ToList();
observableCache.Sort(new Comparison <T>(u_comparer));
var str = observableCache.ToObservable()
.ToAtemporalStreamable(TimelinePolicy.Sequence(80000))
;
var str2 = str
.SetProperty().IsConstantDuration(true, StreamEvent.InfinitySyncTime)
;
var str3 = str2.Cache();
return(str3);
}
public static void YahooBenchmarkTest(IPerfTestState state)
{
var rowCount = 0UL;
var batchCount = 0UL;
var eventArray = new StreamEvent <Event> [ArraySize];
var segment = new ArraySegment <StreamEvent <Event> >(eventArray);
var events = new Subject <ArraySegment <StreamEvent <Event> > >();
var inputStream = events.ToStreamable();
var outputStream = BenchmarkQuery(inputStream);
outputStream.ToStreamEventObservable().ForEachAsync(o => rowCount++);
var stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 0; i < 1000; i++)
{
PopulateArray(eventArray);
eventArray[ArraySize - 1] = StreamEvent.CreatePunctuation <Event>(DateTime.Now.Ticks);
events.OnNext(segment);
batchCount++;
if (batchCount % 100 == 0)
{
Console.WriteLine(new { rowCount, batchCount, time = stopwatch.ElapsedMilliseconds });
}
}
}
private static void JoinIntervalsTest(bool fixedInterval = false)
{
const long duration = 10;
var input1 = new StreamEvent <string>[]
{
StreamEvent.CreateInterval(100, 100 + duration, "A1"),
StreamEvent.CreateInterval(101, 101 + duration, "B1"),
StreamEvent.CreateInterval(102, 102 + duration, "C1"),
StreamEvent.CreateInterval(120, 120 + duration, "A3"),
StreamEvent.CreateInterval(121, 121 + duration, "B3"),
StreamEvent.CreateInterval(122, 122 + duration, "C3"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var input2 = new[]
{
StreamEvent.CreateInterval(105, 105 + duration, "A2"),
StreamEvent.CreateInterval(106, 106 + duration, "B2"),
StreamEvent.CreateInterval(107, 107 + duration, "D2"),
StreamEvent.CreateInterval(125, 125 + duration, "A4"),
StreamEvent.CreateInterval(126, 126 + duration, "B4"),
StreamEvent.CreateInterval(127, 127 + duration, "C4"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var inputStream1 = input1.ToStreamable();
var inputStream2 = input2.ToStreamable();
if (fixedInterval)
{
inputStream1 = inputStream1.AlterEventDuration(duration);
inputStream2 = inputStream2.AlterEventDuration(duration);
}
var outputStream = inputStream1.Join(
inputStream2,
l => (l != null ? l[0].ToString() : null),
r => (r != null ? r[0].ToString() : null),
(l, r) => $"{l},{r}");
var correct = new[]
{
StreamEvent.CreateInterval(105, 110, "A1,A2"),
StreamEvent.CreateInterval(106, 111, "B1,B2"),
StreamEvent.CreateInterval(125, 130, "A3,A4"),
StreamEvent.CreateInterval(126, 131, "B3,B4"),
StreamEvent.CreateInterval(127, 132, "C3,C4"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(outputStream.IsEquivalentTo(correct));
}
public void TestTopKRobsBug()
{
/*
* Time: 1 2 3 4 5
* Input: 10 |-----------|
* 20 |-----|
* 30 |--|
* 40 |-----|
* 50 |--|
* Output: 20 40 50 10
* 10 30 40
* 20 10
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 10),
StreamEvent.CreateInterval(1, 3, 20),
StreamEvent.CreateInterval(2, 3, 30),
StreamEvent.CreateInterval(2, 4, 40),
StreamEvent.CreateInterval(3, 4, 50),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Aggregate(a => a.TopK(v => v, 3), b => b.Count(), (a, b) => a.Select(v => v.Payload)).SelectMany(v => v);
var correct = new[]
{
StreamEvent.CreateStart(1, 20),
StreamEvent.CreateEnd(2, 1, 20),
StreamEvent.CreateStart(1, 10),
StreamEvent.CreateEnd(2, 1, 10),
StreamEvent.CreateStart(2, 40),
StreamEvent.CreateEnd(3, 2, 40),
StreamEvent.CreateStart(2, 30),
StreamEvent.CreateEnd(3, 2, 30),
StreamEvent.CreateStart(2, 20),
StreamEvent.CreateEnd(3, 2, 20),
StreamEvent.CreateStart(3, 50),
StreamEvent.CreateEnd(4, 3, 50),
StreamEvent.CreateStart(3, 40),
StreamEvent.CreateEnd(4, 3, 40),
StreamEvent.CreateStart(3, 10),
StreamEvent.CreateEnd(4, 3, 10),
StreamEvent.CreateStart(4, 10),
StreamEvent.CreateEnd(5, 4, 10),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestGroupAggregate()
{
var input = new[]
{
StreamEvent.CreateStart(0, 100),
StreamEvent.CreateStart(0, 105),
StreamEvent.CreateStart(0, 104),
StreamEvent.CreateStart(0, 200),
StreamEvent.CreateStart(0, 201),
StreamEvent.CreateStart(0, 300),
StreamEvent.CreateStart(0, 302),
StreamEvent.CreateStart(0, 303),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.GroupAggregate(
a => a / 100,
b => b.Count(),
c => c.Sum(v => v % 100),
(key, count, sum) => key.Key * 100 + (int)count * 10 + sum);
// Also test max supported number of aggs (15).
var output2 = input.GroupAggregate(
a => a / 100,
b => b.Count(),
c => c.Sum(v => v % 100),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
b => b.Count(),
(key, c, sum, d, e, f, g, h, i, j, k, l, m, n, o, p) =>
key.Key * 100 + (int)((c + d + e + f + g + h + i + j + k + l + m + n + o) / 2) * 10 + sum);
var correct = new[]
{
StreamEvent.CreateStart(0, 139),
StreamEvent.CreateStart(0, 221),
StreamEvent.CreateStart(0, 335),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
private static void LeftAntiSemiJoinIntervalsTest()
{
var input1 = new[]
{
StreamEvent.CreateInterval(100, 101, "A3"),
StreamEvent.CreateInterval(100, 110, "A1"),
StreamEvent.CreateInterval(101, 105, "A2"),
StreamEvent.CreateInterval(109, 112, "A4"),
StreamEvent.CreateInterval(102, 112, "B1"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var input2 = new[]
{
StreamEvent.CreateInterval(101, 103, "A"),
StreamEvent.CreateInterval(106, 108, "A"),
StreamEvent.CreateInterval(106, 109, "A"),
StreamEvent.CreateInterval(100, 110, "C"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
var inputStream1 = input1.ToCleanStreamable();
var inputStream2 = input2.ToCleanStreamable();
var outputStream = inputStream1.WhereNotExists(
inputStream2,
l => (l != null ? l[0].ToString() : null),
r => (r != null ? r[0].ToString() : null));
var correct = new[]
{
StreamEvent.CreateStart(100, "A1"),
StreamEvent.CreateEnd(101, 100, "A1"),
StreamEvent.CreateStart(103, "A1"),
StreamEvent.CreateEnd(106, 103, "A1"),
StreamEvent.CreateInterval(109, 110, "A1"),
StreamEvent.CreateInterval(103, 105, "A2"),
StreamEvent.CreateInterval(100, 101, "A3"),
StreamEvent.CreateInterval(109, 112, "A4"),
StreamEvent.CreateStart(102, "B1"),
StreamEvent.CreateEnd(112, 102, "B1"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(outputStream.IsEquivalentTo(correct));
}
private static void ArrayBasedIngressExample()
{
// Create the first array segment.
StreamEvent <Point>[] values1 =
{
StreamEvent.CreateInterval(1, 10, new Point {
x = 1, y = 2
}),
StreamEvent.CreateInterval(2, 10, new Point {
x = 2, y = 4
}),
StreamEvent.CreateInterval(3, 10, new Point {
x = 3, y = 6
}),
StreamEvent.CreateInterval(4, 10, new Point {
x = 4, y = 8
}),
StreamEvent.CreateInterval(5, 10, new Point {
x = 5, y = 10
})
};
// Create the second array segment.
StreamEvent <Point>[] values2 =
{
StreamEvent.CreateInterval(6, 10, new Point {
x = 6, y = 12
}),
StreamEvent.CreateInterval(7, 10, new Point {
x = 7, y = 14
}),
StreamEvent.CreateInterval(8, 10, new Point {
x = 8, y = 16
}),
StreamEvent.CreateInterval(9, 10, new Point {
x = 9, y = 18
}),
StreamEvent.CreatePunctuation <Point>(StreamEvent.InfinitySyncTime)
};
var segment1 = new ArraySegment <StreamEvent <Point> >(values1);
var segment2 = new ArraySegment <StreamEvent <Point> >(values2);
var segments = new ArraySegment <StreamEvent <Point> >[] { segment1, segment2 };
// Array-based ingress.
var input = segments.ToObservable().ToStreamable(OnCompletedPolicy.None());
Console.WriteLine("Input =");
input.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
Console.ReadLine();
}
public static IStreamable <Empty, T> ToStatStreamable <T>(this IEnumerable <T> source)
{
var batchSize = 80000;
var timeStamps = StreamEvent.MinSyncTime.CountToInfinity().SelectMany(i => Enumerable.Repeat(i, batchSize));
var streamEventEnumerable = Enumerable.Zip(timeStamps, source, (t, e) => StreamEvent.CreateStart(t, e))
.Concat(new StreamEvent <T>[] { StreamEvent.CreatePunctuation <T>(StreamEvent.InfinitySyncTime), });
var stream = streamEventEnumerable
.ToObservable(Scheduler.Default)
.ToStreamable(null, FlushPolicy.FlushOnPunctuation, PeriodicPunctuationPolicy.Time(1))
.SetProperty().IsConstantDuration(true, StreamEvent.InfinitySyncTime);
return(stream);
}
private void OnCompleted(Subject <StreamEvent <int> > inputSubject)
{
// Add expectations
if (this.completedPolicy != OnCompletedPolicy.None)
{
long time = this.completedPolicy == OnCompletedPolicy.EndOfStream ? StreamEvent.InfinitySyncTime : IngressEventCount - 1;
this.expectedBatch.Enqueue(StreamEvent.CreatePunctuation <int>(time));
MoveExpectedBatchToOutput();
}
// Ingress
inputSubject.OnCompleted();
}
/*
* Mostly a normal blocking socket listener
* calls observer.onNext()
*/
public void StartListener()
{
byte[] bytes;
IPAddress ipAddress = IPAddress.Parse(ip);
IPEndPoint localEndpoint = new IPEndPoint(ipAddress, port);
observers = new List <IObserver <StreamEvent <Payload> > >();
// make blocking socket server
Socket server = new Socket(ipAddress.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
Console.WriteLine(localEndpoint.ToString());
try {
server.Bind(localEndpoint);
server.Listen(10);
while (true)
{
Console.WriteLine("Waiting for a connection...");
Socket handler = server.Accept();
string bytesString = null;
bytes = new byte[1024];
while (true)
{
int bytesRec = handler.Receive(bytes);
if (bytesRec > 0)
{
bytesString += Encoding.ASCII.GetString(bytes, 0, bytesRec);
}
else
{
break;
}
}
// Show the data on the console.
foreach (IObserver <StreamEvent <Payload> > observer in observers)
{
Payload payload = ParseInput(bytesString);
observer.OnNext(StreamEvent.CreatePoint(payload.Timestamp.Ticks, payload));
observer.OnNext(StreamEvent.CreatePunctuation <Payload>(payload.Timestamp.Ticks + 2));
}
}
} catch (Exception e) {
Console.WriteLine(e.ToString());
}
}
private static void WhereExample10()
{
StreamEvent <ValueTuple <int, int> >[] values2 =
{
StreamEvent.CreateInterval(1, 10, new ValueTuple <int, int> {
Item1 = 1, Item2 = 2
}),
StreamEvent.CreateInterval(2, 10, new ValueTuple <int, int> {
Item1 = 2, Item2 = 4
}),
StreamEvent.CreateInterval(3, 10, new ValueTuple <int, int> {
Item1 = 3, Item2 = 6
}),
StreamEvent.CreateInterval(4, 10, new ValueTuple <int, int> {
Item1 = 4, Item2 = 8
}),
StreamEvent.CreateInterval(5, 10, new ValueTuple <int, int> {
Item1 = 5, Item2 = 10
}),
StreamEvent.CreateInterval(6, 10, new ValueTuple <int, int> {
Item1 = 6, Item2 = 12
}),
StreamEvent.CreateInterval(7, 10, new ValueTuple <int, int> {
Item1 = 7, Item2 = 14
}),
StreamEvent.CreateInterval(8, 10, new ValueTuple <int, int> {
Item1 = 8, Item2 = 16
}),
StreamEvent.CreateInterval(9, 10, new ValueTuple <int, int> {
Item1 = 9, Item2 = 18
}),
StreamEvent.CreatePunctuation <ValueTuple <int, int> >(StreamEvent.InfinitySyncTime)
};
var input = values2.ToObservable().ToStreamable(OnCompletedPolicy.None());
Console.WriteLine("Input =");
input.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
// Apply a filter on one field.
Console.WriteLine();
Console.WriteLine("Query: input.Where(p => p.x > 5)");
var output = input.Where(p => p.Item1 > 5);
Console.WriteLine();
Console.WriteLine("Output =");
output.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
}
public static StreamEvent <int>[] Get()
{
return(new[]
{
StreamEvent.CreateInterval(1, 10, 1),
StreamEvent.CreateInterval(2, 10, 2),
StreamEvent.CreateInterval(3, 10, 3),
StreamEvent.CreateInterval(4, 10, 4),
StreamEvent.CreateInterval(5, 10, 5),
StreamEvent.CreateInterval(6, 10, 6),
StreamEvent.CreateInterval(7, 10, 7),
StreamEvent.CreateInterval(8, 10, 8),
StreamEvent.CreateInterval(9, 10, 9),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
});
}
private void EquiJoinWorker(long?constantDuration = null)
{
var left = new[]
{
StreamEvent.CreateStart(0, 100),
StreamEvent.CreateStart(0, 110),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var right = new[]
{
StreamEvent.CreateStart(0, 200),
StreamEvent.CreateStart(0, 210),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
if (constantDuration.HasValue)
{
left = left.AlterEventDuration(constantDuration.Value);
right = right.AlterEventDuration(constantDuration.Value);
}
var output = new List <StreamEvent <(int, int)> >();
left.Join(right, (l, r) => CreateValueTuple(l, r))
.ToStreamEventObservable()
.ForEachAsync(e => output.Add(e))
.Wait();
var correct = new[]
{
StreamEvent.CreateStart(0, ValueTuple.Create(110, 200)),
StreamEvent.CreateStart(0, ValueTuple.Create(100, 200)),
StreamEvent.CreateStart(0, ValueTuple.Create(110, 210)),
StreamEvent.CreateStart(0, ValueTuple.Create(100, 210)),
StreamEvent.CreatePunctuation <ValueTuple <int, int> >(StreamEvent.InfinitySyncTime)
};
if (constantDuration.HasValue && constantDuration.Value != StreamEvent.InfinitySyncTime)
{
correct = correct
.Select(e => e.IsPunctuation ? e : StreamEvent.CreateInterval(e.StartTime, e.StartTime + constantDuration.Value, e.Payload))
.ToArray();
}
Assert.IsTrue(correct.SequenceEqual(output));
}
private static void ArrayBasedEgressExample()
{
StreamEvent <Point>[] values =
{
StreamEvent.CreateInterval(1, 10, new Point {
x = 1, y = 2
}),
StreamEvent.CreateInterval(2, 10, new Point {
x = 2, y = 4
}),
StreamEvent.CreateInterval(3, 10, new Point {
x = 3, y = 6
}),
StreamEvent.CreateInterval(4, 10, new Point {
x = 4, y = 8
}),
StreamEvent.CreateInterval(5, 10, new Point {
x = 5, y = 10
}),
StreamEvent.CreateInterval(6, 10, new Point {
x = 6, y = 12
}),
StreamEvent.CreateInterval(7, 10, new Point {
x = 7, y = 14
}),
StreamEvent.CreateInterval(8, 10, new Point {
x = 8, y = 16
}),
StreamEvent.CreateInterval(9, 10, new Point {
x = 9, y = 18
}),
StreamEvent.CreatePunctuation <Point>(StreamEvent.InfinitySyncTime)
};
var input = values.ToObservable().ToStreamable(OnCompletedPolicy.None());
Console.WriteLine("Input =");
input.ToStreamEventObservable().ForEachAsync(e => Console.WriteLine(e)).Wait();
// Array-based egress.
Console.WriteLine();
Console.WriteLine("Output =");
input.ToStreamEventArrayObservable().ForEachAsync(
e => e.ToObservable().ForEachAsync(w => Console.WriteLine(w))).Wait();
Console.ReadLine();
}
public void PercentileContAggregate()
{
/*
* Time: 1 2 3 4 5
* Input: 10 |-----------|
* 20 |-----|
* 30 |--|
* 40 |-----|
* 50 |--|
* Output: 16 28 42 10
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 10),
StreamEvent.CreateInterval(1, 3, 20),
StreamEvent.CreateInterval(2, 3, 30),
StreamEvent.CreateInterval(2, 4, 40),
StreamEvent.CreateInterval(3, 4, 50),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var correct = new[]
{
StreamEvent.CreateStart(1, 16.0),
StreamEvent.CreateEnd(2, 1, 16.0),
StreamEvent.CreateStart(2, 28.0),
StreamEvent.CreateEnd(3, 2, 28.0),
StreamEvent.CreateStart(3, 42.0),
StreamEvent.CreateEnd(4, 3, 42.0),
StreamEvent.CreateStart(4, 10.0),
StreamEvent.CreateEnd(5, 4, 10.0),
StreamEvent.CreatePunctuation <double>(StreamEvent.InfinitySyncTime)
};
var output = input.Aggregate(w => w.PercentileContinuous(0.6, e => e));
Assert.IsTrue(output.IsEquivalentTo(correct));
// And with reverse order...
output = input.Aggregate(w => w.PercentileContinuous((x, y) => y.CompareTo(x), 0.4, e => e));
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestWhereAggregate()
{
/*
* Time: 1 2 3 4 5
* Input: 10 |-------|
* 20 |---|
* 30 |-|
* 40 |---|
* 50 |-|
* Output: 2 4 3 1 0
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 10),
StreamEvent.CreateInterval(1, 3, 20),
StreamEvent.CreateInterval(2, 3, 30),
StreamEvent.CreateInterval(2, 4, 40),
StreamEvent.CreateInterval(3, 4, 50),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Aggregate(
a => a.Count(),
b => b.Where(v => v >= 30).Count(),
b => b.Where(v => v >= 30).Where(v => v <= 40).Count(),
(a, b, c) => (a * 100) + (b * 10) + c);
var correct = new[]
{
StreamEvent.CreateStart <ulong>(1, 200),
StreamEvent.CreateEnd <ulong>(2, 1, 200),
StreamEvent.CreateStart <ulong>(2, 422),
StreamEvent.CreateEnd <ulong>(3, 2, 422),
StreamEvent.CreateStart <ulong>(3, 321),
StreamEvent.CreateEnd <ulong>(4, 3, 321),
StreamEvent.CreateStart <ulong>(4, 100),
StreamEvent.CreateEnd <ulong>(5, 4, 100),
StreamEvent.CreatePunctuation <ulong>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestAverageAggregate_Double()
{
/*
* Time: 1 2 3 4 5
* Input: 1 |---------------|
* 2 |-------|
* 4 |---|
* 8 |-------|
* 16 |---|
* Output: 3 15 25 1
* - -- -- -
* 2 4 3 1
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 1d),
StreamEvent.CreateInterval(1, 3, 2d),
StreamEvent.CreateInterval(2, 3, 4d),
StreamEvent.CreateInterval(2, 4, 8d),
StreamEvent.CreateInterval(3, 4, 16d),
StreamEvent.CreatePunctuation <double>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Average(v => v);
var correct = new[]
{
StreamEvent.CreateStart(1, 3.0 / 2),
StreamEvent.CreateEnd(2, 1, 3.0 / 2),
StreamEvent.CreateStart(2, 15.0 / 4),
StreamEvent.CreateEnd(3, 2, 15.0 / 4),
StreamEvent.CreateStart(3, 25.0 / 3),
StreamEvent.CreateEnd(4, 3, 25.0 / 3),
StreamEvent.CreateStart(4, 1.0),
StreamEvent.CreateEnd(5, 4, 1.0),
StreamEvent.CreatePunctuation <double>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void DisjointUnionPunctuations()
{
var left = new Subject <StreamEvent <int> >();
var right = new Subject <StreamEvent <int> >();
var qc = new QueryContainer();
var leftInput = qc.RegisterInput(left);
var rightInput = qc.RegisterInput(right);
var actualOutput = new List <StreamEvent <int> >();
var union = new MultiUnionStreamable <Empty, int>(new IStreamable <Empty, int>[] { leftInput, rightInput }, guaranteedDisjoint: true);
var egress = qc.RegisterOutput(union).ForEachAsync(o => actualOutput.Add(o));
var process = qc.Restore();
left.OnNext(StreamEvent.CreatePoint(100, 1));
left.OnNext(StreamEvent.CreatePunctuation <int>(101));
right.OnNext(StreamEvent.CreatePoint(100, 1));
right.OnNext(StreamEvent.CreatePunctuation <int>(110));
process.Flush();
left.OnNext(StreamEvent.CreatePoint(101, 1));
right.OnNext(StreamEvent.CreatePoint(110, 1));
process.Flush();
left.OnCompleted();
right.OnCompleted();
var expected = new StreamEvent <int>[]
{
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePunctuation <int>(101),
StreamEvent.CreatePoint(101, 1),
StreamEvent.CreatePoint(110, 1),
StreamEvent.CreatePunctuation <int>(110),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(actualOutput));
}
public void TestCountNotNullAggregate_AllNulls()
{
/*
* Time: 1 2 3 4 5
* Input: 10 |-------|
* 20 |---|
* 30 |-|
* 40 |---|
* 50 |-|
* Output: 0 0 0 0 0
*/
var input = new[]
{
StreamEvent.CreateInterval <string>(1, 5, null),
StreamEvent.CreateInterval <string>(1, 3, null),
StreamEvent.CreateInterval <string>(2, 3, null),
StreamEvent.CreateInterval <string>(2, 4, null),
StreamEvent.CreateInterval <string>(3, 4, null),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.CountNotNull(v => v);
var correct = new[]
{
StreamEvent.CreateStart <ulong>(1, 0),
StreamEvent.CreateEnd <ulong>(2, 1, 0),
StreamEvent.CreateStart <ulong>(2, 0),
StreamEvent.CreateEnd <ulong>(3, 2, 0),
StreamEvent.CreateStart <ulong>(3, 0),
StreamEvent.CreateEnd <ulong>(4, 3, 0),
StreamEvent.CreateStart <ulong>(4, 0),
StreamEvent.CreateEnd <ulong>(5, 4, 0),
StreamEvent.CreatePunctuation <ulong>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestMaxAggregate()
{
/*
* Time: 1 2 3 4 5
* Input: 10 |-----------|
* 20 |-----|
* 30 |--|
* 40 |-----|
* 50 |--|
* Output: 10 40 50 10 0
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 10),
StreamEvent.CreateInterval(1, 3, 20),
StreamEvent.CreateInterval(2, 3, 30),
StreamEvent.CreateInterval(2, 4, 40),
StreamEvent.CreateInterval(3, 4, 50),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Max(v => v);
var correct = new[]
{
StreamEvent.CreateStart(1, 20),
StreamEvent.CreateEnd(2, 1, 20),
StreamEvent.CreateStart(2, 40),
StreamEvent.CreateEnd(3, 2, 40),
StreamEvent.CreateStart(3, 50),
StreamEvent.CreateEnd(4, 3, 50),
StreamEvent.CreateStart(4, 10),
StreamEvent.CreateEnd(5, 4, 10),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestSumAggregate_Long()
{
/*
* Time: 1 2 3 4 5
* Input: 1 |---------------|
* 2 |-------|
* 4 |---|
* 8 |-------|
* 16 |---|
* Output: 3 15 25 1 0
*/
var input = new[]
{
StreamEvent.CreateInterval <long>(1, 5, 1),
StreamEvent.CreateInterval <long>(1, 3, 2),
StreamEvent.CreateInterval <long>(2, 3, 4),
StreamEvent.CreateInterval <long>(2, 4, 8),
StreamEvent.CreateInterval <long>(3, 4, 16),
StreamEvent.CreatePunctuation <long>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Sum(v => v);
var correct = new[]
{
StreamEvent.CreateStart <long>(1, 3),
StreamEvent.CreateEnd <long>(2, 1, 3),
StreamEvent.CreateStart <long>(2, 15),
StreamEvent.CreateEnd <long>(3, 2, 15),
StreamEvent.CreateStart <long>(3, 25),
StreamEvent.CreateEnd <long>(4, 3, 25),
StreamEvent.CreateStart <long>(4, 1),
StreamEvent.CreateEnd <long>(5, 4, 1),
StreamEvent.CreatePunctuation <long>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestCountAggregate_WithNulls()
{
/*
* Time: 1 2 3 4 5
* Input: 10 |-------|
* 20 |---|
* 30 |-|
* 40 |---|
* 50 |-|
* Output: 2 4 3 1 0
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, "10"),
StreamEvent.CreateInterval <string>(1, 3, null),
StreamEvent.CreateInterval(2, 3, "30"),
StreamEvent.CreateInterval(2, 4, "40"),
StreamEvent.CreateInterval(3, 4, "50"),
StreamEvent.CreatePunctuation <string>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Count();
var correct = new[]
{
StreamEvent.CreateStart <ulong>(1, 2),
StreamEvent.CreateEnd <ulong>(2, 1, 2),
StreamEvent.CreateStart <ulong>(2, 4),
StreamEvent.CreateEnd <ulong>(3, 2, 4),
StreamEvent.CreateStart <ulong>(3, 3),
StreamEvent.CreateEnd <ulong>(4, 3, 3),
StreamEvent.CreateStart <ulong>(4, 1),
StreamEvent.CreateEnd <ulong>(5, 4, 1),
StreamEvent.CreatePunctuation <ulong>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestSumAggregate_Decimal()
{
/*
* Time: 1 2 3 4 5
* Input: 1 |---------------|
* 2 |-------|
* 4 |---|
* 8 |-------|
* 16 |---|
* Output: 3 15 25 1 0
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 1.5m),
StreamEvent.CreateInterval(1, 3, 2m),
StreamEvent.CreateInterval(2, 3, 3.6m),
StreamEvent.CreateInterval(2, 4, 8.2m),
StreamEvent.CreateInterval(3, 4, 16m),
StreamEvent.CreatePunctuation <decimal>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Sum(v => v);
var correct = new[]
{
StreamEvent.CreateStart(1, 3.5m),
StreamEvent.CreateEnd(2, 1, 3.5m),
StreamEvent.CreateStart(2, 15.3m),
StreamEvent.CreateEnd(3, 2, 15.3m),
StreamEvent.CreateStart(3, 25.7m),
StreamEvent.CreateEnd(4, 3, 25.7m),
StreamEvent.CreateStart(4, 1.5m),
StreamEvent.CreateEnd(5, 4, 1.5m),
StreamEvent.CreatePunctuation <decimal>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void TestAggregateByKey3()
{
var input = new[]
{
StreamEvent.CreateStart(0, 100),
StreamEvent.CreateStart(0, 105),
StreamEvent.CreateStart(0, 104),
StreamEvent.CreateStart(0, 200),
StreamEvent.CreateStart(0, 201),
StreamEvent.CreateStart(0, 300),
StreamEvent.CreateStart(0, 302),
StreamEvent.CreateStart(0, 303),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable().SetProperty().IsConstantDuration(true, StreamEvent.InfinitySyncTime);
var output = input
.Select(e => new StructTuple <long, long> {
Item1 = 0, Item2 = e
})
.AggregateByKey(
a => (int)a.Item2 / 100,
b => b.Sum(s => (int)(s.Item2 & 0xffff)),
c => c.Count(),
(sum, count) => new StructTuple <int, ulong> {
Item1 = sum, Item2 = count * 34
},
(key, agg) => new StructTuple <int, ulong, ulong> {
Item1 = key, Item2 = (ulong)key * 100 + (ulong)agg.Item1 * 10, Item3 = (ulong)key * 100 + (ulong)agg.Item2 * 0
})
.Select(e => (int)e.Item2);
var correct = new[]
{
StreamEvent.CreateStart(0, 3190),
StreamEvent.CreateStart(0, 4210),
StreamEvent.CreateStart(0, 9350),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void Stitch_PointsCodegen()
{
StreamEvent <int>[] input = new[]
{
StreamEvent.CreatePoint(100, 1),
StreamEvent.CreatePoint(102, 1),
};
IList <StreamEvent <int> > result = Execute(input, x => x.Stitch());
StreamEvent <int>[] expected = new[]
{
StreamEvent.CreateStart(100, 1),
StreamEvent.CreateEnd(101, 100, 1),
StreamEvent.CreateStart(102, 1),
StreamEvent.CreateEnd(103, 102, 1),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime),
};
Assert.IsTrue(expected.SequenceEqual(result));
}