public void StitchTestDictionaryCodegen()
{
// We have done horrible thigns to the dictionary: all elements are
// identical!
var inputList = new[] {
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(2, 1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("B", 3)),
StreamEvent.CreateEnd(3, 2, StructTuple.Create("B", 3)),
StreamEvent.CreateStart(3, StructTuple.Create("B", 3)),
StreamEvent.CreateEnd(4, 3, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(4, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 4, StructTuple.Create("B", 3))
};
var compareTo = new[] {
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 1, StructTuple.Create("A", 3)),
END
};
var input = inputList.ToObservable().ToStreamable()
.SetProperty()
.PayloadEqualityComparer(new EqualityComparerExpression <StructTuple <string, int> >(
(st1, st2) => (st1.Item1 == st2.Item1) ||
(st1.Item1 == "A" && st2.Item1 == "B") ||
(st1.Item1 == "B" && st2.Item1 == "A"),
str => 7));
var outputStream = input.Stitch();
var output = outputStream.ToStreamEventObservable(ReshapingPolicy.None).ToEnumerable().ToArray();
Assert.IsTrue(output.SequenceEqual(compareTo));
}
public void TestAggregateByKeyTupleDecomposition()
{
var input = new[]
{
StreamEvent.CreateStart(0, 100),
StreamEvent.CreateStart(0, 105),
StreamEvent.CreateStart(0, 104),
StreamEvent.CreateStart(0, 200),
StreamEvent.CreateStart(0, 201),
StreamEvent.CreateStart(0, 300),
StreamEvent.CreateStart(0, 302),
StreamEvent.CreateStart(0, 303),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = new List <StreamEvent <(bool, ulong)> >();
input
.GroupAggregate(s => true, w => w.Count(), (g, c) => ValueTuple.Create(g.Key, c))
.ToStreamEventObservable()
.ForEachAsync(e => output.Add(e))
.Wait();
var correct = new[]
{
StreamEvent.CreateStart(0, ValueTuple.Create(true, (ulong)8)),
StreamEvent.CreatePunctuation <ValueTuple <bool, ulong> >(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(correct.SequenceEqual(output));
}
public void StitchTestSimpleSplice()
{
// nothing interesting happens here
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 4, "A")
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(5, 1, "A"),
END
};
var input = inputList.ToObservable().ToStreamable();
var outputStream = input.Stitch();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public static void Main(string[] args)
{
// Poll performance counter 4 times a second
var pollingInterval = TimeSpan.FromSeconds(0.25);
// Take the total processor utilization performance counter
string categoryName = "Processor";
string counterName = "% Processor Time";
string instanceName = "_Total";
// Create an observable that feeds the performance counter periodically
IObservable <PerformanceCounterSample> source =
new PerformanceCounterObservable(categoryName, counterName, instanceName, pollingInterval);
// Load the observable as a stream in Trill, injecting a punctuation every second. Because we use
// FlushPolicy.FlushOnPunctuation, this will also flush the data every second.
var inputStream =
source.Select(e => StreamEvent.CreateStart(e.StartTime.Ticks, e))
.ToStreamable(
null,
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time((ulong)TimeSpan.FromSeconds(1).Ticks));
// The query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query = inputStream.AlterEventDuration(windowSize).Average(e => e.Value);
// Egress results and write to console
query.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
public void PointAtEndTest3()
{
// nothing interesting happens here
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 4, "A")
};
var compareTo = new[]
{
StreamEvent.CreatePoint(101, "A"),
StreamEvent.CreatePoint(102, "A"),
StreamEvent.CreatePoint(103, "A"),
StreamEvent.CreatePoint(104, "A"),
END
};
var input = inputList.ToList().ToObservable().ToStreamable()
.AlterEventDuration(100);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public void PointAtEndTest4()
{
var payload = StructTuple.Create("A", 3);
// nothing interesting happens here
var inputList = new[] {
StreamEvent.CreateStart(1, payload),
StreamEvent.CreateEnd(2, 1, payload),
StreamEvent.CreateStart(2, payload),
StreamEvent.CreateEnd(3, 2, payload),
StreamEvent.CreateStart(3, payload),
StreamEvent.CreateEnd(4, 3, payload),
StreamEvent.CreateStart(4, payload),
StreamEvent.CreateEnd(5, 4, payload)
};
var compareTo = new StreamEvent <StructTuple <string, int> >[] {
StreamEvent.CreatePoint(2, payload),
StreamEvent.CreatePoint(3, payload),
StreamEvent.CreatePoint(4, payload),
StreamEvent.CreatePoint(5, payload),
END2
};
var input = inputList.ToList().ToObservable().ToStreamable().SetProperty().IsIntervalFree(true);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.Select(r => r.Item1).IsEquivalentTo(compareTo.Select(r => r.IsData ? StreamEvent.CreatePoint(r.SyncTime, r.Payload.Item1) : END).ToArray()));
}
public void PointAtEndTest1()
{
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 4, "A")
};
var compareTo = new[]
{
StreamEvent.CreatePoint(2, "A"),
StreamEvent.CreatePoint(3, "A"),
StreamEvent.CreatePoint(4, "A"),
StreamEvent.CreatePoint(5, "A"),
END
};
var input = inputList.ToList().ToObservable().ToStreamable().SetProperty().IsIntervalFree(true);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public static void Test()
{
// Poll performance counter 4 times a second
TimeSpan pollingInterval = TimeSpan.FromSeconds(0.25);
// Take the total processor utilization performance counter
string categoryName = "Processor";
string counterName = "% Processor Time";
string instanceName = "_Total";
// Create an observable that feeds the performance counter periodically
IObservable <PerformanceCounterSample> source = new PerformanceCounterObservable(categoryName, counterName, instanceName, pollingInterval);
// Load the observable as a stream in Trill
var inputStream =
source.Select(e => StreamEvent.CreateStart(e.StartTime.Ticks, e)) // Create an IObservable of StreamEvent<>
.ToStreamable(/*OnCompletedPolicy.Flush(), PeriodicPunctuationPolicy.Count(4)*/); // Create a streamable with a punctuation every 4 events
// The query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query = inputStream.AlterEventDuration(windowSize).Average(e => e.Value);
// Egress results and write to console
query.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
public void TestSumAggregate_Double()
{
/*
* Time: 1 2 3 4 5
* Input: 1 |---------------|
* 2 |-------|
* 4 |---|
* 8 |-------|
* 16 |---|
* Output: 3 15 25 1 0
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 1.0),
StreamEvent.CreateInterval(1, 3, 2.0),
StreamEvent.CreatePunctuation <double>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Sum(v => v);
var correct = new[]
{
StreamEvent.CreateStart(1, 3.0),
StreamEvent.CreateEnd(3, 1, 3.0),
StreamEvent.CreateStart(3, 1.0),
StreamEvent.CreateEnd(5, 3, 1.0),
StreamEvent.CreatePunctuation <double>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public static IStreamable <Empty, T> ToStreamable <T>(
this IEnumerable <T> enumerable,
int startEdge = 0)
=> enumerable
.ToObservable()
.Select(e => StreamEvent.CreateStart(startEdge, e))
.ToStreamable();
public void ReorderTest2ColumnarSmallBatch()
{
var oldSortingTechnique = Config.IngressSortingTechnique;
Config.IngressSortingTechnique = SortingTechnique.PriorityQueue;
var outputList = new List <StreamEvent <int> >();
double disorderFraction = 0.5;
int reorderLatency = 202;
int disorderAmount = 200;
var rand = new Random(2);
var disorderedData =
Enumerable.Range(disorderAmount, 500000).ToList()
.Select(e => StreamEvent.CreateStart(rand.NextDouble() < disorderFraction ? e - rand.Next(0, disorderAmount) : e, 0))
.ToList();
var stream = disorderedData.ToObservable().ToStreamable(DisorderPolicy.Drop(reorderLatency));
stream.ToStreamEventObservable().ForEachAsync(e => { if (e.IsData)
{
outputList.Add(e);
}
}).Wait();
disorderedData.Sort((a, b) => a.SyncTime.CompareTo(b.SyncTime));
Assert.IsTrue(disorderedData.SequenceEqual(outputList));
Config.IngressSortingTechnique = oldSortingTechnique;
}
public void StitchTestPassthroughCodegen()
{
var inputList = new List <StreamEvent <StructTuple <string, int> > >
{
// nothing interesting happens here
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("B", 3)),
StreamEvent.CreateEnd(3, 1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(4, 2, StructTuple.Create("B", 3))
};
var input = inputList.ToObservable().ToStreamable();
var outputStream = input.Stitch();
var compareTo = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("B", 3)),
StreamEvent.CreateEnd(3, 1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(4, 2, StructTuple.Create("B", 3)),
END
};
var output = outputStream.ToStreamEventObservable(ReshapingPolicy.None).ToEnumerable().ToArray();
Assert.IsTrue(output.SequenceEqual(compareTo));
}
public void StitchTestSimpleSpliceCodegen()
{
// nothing interesting happens here
var inputList = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(2, 1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(3, 2, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(3, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(4, 3, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(4, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 4, StructTuple.Create("A", 3))
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 1, StructTuple.Create("A", 3)),
END
};
var input = inputList.ToObservable().ToStreamable();
var outputStream = input.Stitch();
var output = outputStream.ToStreamEventObservable(ReshapingPolicy.None).ToEnumerable().ToArray();
Assert.IsTrue(output.SequenceEqual(compareTo));
}
public static void StitchTestMultisetMultiStart()
{
// nothing interesting happens here
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 1, "A"),
StreamEvent.CreateEnd(6, 5, "A"),
StreamEvent.CreateEnd(7, 4, "A")
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(6, 1, "A"), // 6->5->1
StreamEvent.CreateEnd(7, 2, "A"), // 7->4->3->2
END
};
var input = inputList.ToList().ToObservable().ToStreamable();
var outputStream = input.Stitch();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public static void StitchTestMultisetMultiStartCodegen()
{
// nothing interesting happens here
var inputList = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(3, 2, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(3, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(4, 3, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(4, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(6, 5, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(7, 4, StructTuple.Create("A", 3))
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(6, 1, StructTuple.Create("A", 3)), // 6->5->1
StreamEvent.CreateEnd(7, 2, StructTuple.Create("A", 3)), // 7->4->3->2
END
};
var input = inputList.ToList().ToObservable().ToStreamable();
var outputStream = input.Stitch();
var output = outputStream.ToStreamEventObservable(ReshapingPolicy.None).ToEnumerable().ToArray();
Assert.IsTrue(output.SequenceEqual(compareTo));
}
public void StitchTestDictionary()
{
// We have done horrible thigns to the dictionary: all elements are
// identical!
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "B"),
StreamEvent.CreateEnd(3, 2, "B"),
StreamEvent.CreateStart(3, "B"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 4, "B")
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(5, 1, "A"),
END
};
var input = inputList.ToObservable().ToStreamable()
.SetProperty()
.PayloadEqualityComparer(new EqualityComparerExpression <string>(
(str1, str2) => (str1 == str2) ||
(str1 == "A" && str2 == "B") ||
(str1 == "B" && str2 == "A"),
str => 7));
var outputStream = input.Stitch();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public void StitchTestTimeInvert()
{
// This is the odd little case where we swapped "BEGIN" and "END" at both times 3 and 4
// so that they arrive in the opposite order. This shouldn't trouble the system at all:
// Everything that happens at Time 3 is identical. Right? Right!
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateEnd(5, 4, "A")
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(5, 1, "A"),
END
};
var input = inputList.ToObservable().ToStreamable();
var outputStream = input.Stitch();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public void StitchTestMultisetOneStart()
{
// Two different start times
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(2, 1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateEnd(3, 2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateEnd(4, 3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 4, "A"),
StreamEvent.CreateEnd(6, 1, "A")
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateEnd(5, 1, "A"), // 5->4->3->2->1
StreamEvent.CreateEnd(6, 1, "A"), // 6->1
END
};
var input = inputList.ToObservable().ToStreamable();
var outputStream = input.Stitch();
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public void Group6Columnar()
{
var inputEnumerable = new[]
{
StreamEvent.CreateStart(9900, 100),
StreamEvent.CreateStart(10000, 200),
StreamEvent.CreateStart(10020, 300),
StreamEvent.CreateStart(10025, 400),
StreamEvent.CreateStart(10030, 500),
StreamEvent.CreateStart(10040, 600)
};
var inputStream = inputEnumerable.ToObservable().ToStreamable();
var result =
inputStream
.Shard(1)
.Shuffle(e => e)
.SelectKey((k, p) => new { K = k, P = p })
.Unshuffle()
.Unshard()
;
var finalResultSequence = result
.ToStreamEventObservable()
.Where(se => se.IsData)
.Select(se => StreamEvent.CreateStart(se.StartTime, se.Payload.P))
.ToEnumerable()
;
Assert.IsTrue(finalResultSequence.SequenceEqual(inputEnumerable));
}
public void StitchTestMultisetOneStartCodegen()
{
// Two different start times
var inputList = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(2, 1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(3, 2, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(3, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(4, 3, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(4, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 4, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(6, 1, StructTuple.Create("A", 3))
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 1, StructTuple.Create("A", 3)), // 5->4->3->2->1
StreamEvent.CreateEnd(6, 1, StructTuple.Create("A", 3)), // 6->1
END
};
var input = inputList.ToObservable().ToStreamable();
var outputStream = input.Stitch();
var output = outputStream.ToStreamEventObservable(ReshapingPolicy.None).ToEnumerable().ToArray();
Assert.IsTrue(output.SequenceEqual(compareTo));
}
public static StreamCache <Empty, T> Sort <T, U>(this IStreamable <Empty, T> stream, Expression <Func <T, U> > sortBySelector)
{
var u_comparer = Utility.CreateCompoundComparer(sortBySelector, ComparerExpression <U> .Default.GetCompareExpr()).Compile();
IStreamable <Empty, T> stream2 = stream;
stream.GetProperties(out StreamProperties <Empty, T> properties);
if (!properties.IsStartEdgeOnly)
{
var secondStream = new StreamEvent <T>[]
{
StreamEvent.CreateStart(StreamEvent.MaxSyncTime, default(T)),
StreamEvent.CreatePunctuation <T>(StreamEvent.InfinitySyncTime),
}.ToObservable().ToStreamable();
stream2 = stream.Join(secondStream, (x, y) => x);
}
var observableCache = stream2.ToPayloadEnumerable().ToList();
observableCache.Sort(new Comparison <T>(u_comparer));
var str = observableCache.ToObservable()
.ToAtemporalStreamable(TimelinePolicy.Sequence(80000))
;
var str2 = str
.SetProperty().IsConstantDuration(true, StreamEvent.InfinitySyncTime)
;
var str3 = str2.Cache();
return(str3);
}
public void StitchTestTimeInvertCodegen()
{
// This is the odd little case where we swapped "BEGIN" and "END" at both times 3 and 4
// so that they arrive in the opposite order. This shouldn't trouble the system at all:
// Everything that happens at Time 3 is identical. Right? Right!
var inputList = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(2, 1, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(2, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(3, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(3, 2, StructTuple.Create("A", 3)),
StreamEvent.CreateStart(4, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(4, 3, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 4, StructTuple.Create("A", 3))
};
var compareTo = new[]
{
StreamEvent.CreateStart(1, StructTuple.Create("A", 3)),
StreamEvent.CreateEnd(5, 1, StructTuple.Create("A", 3)),
END
};
var input = inputList.ToObservable().ToStreamable();
var outputStream = input.Stitch();
var output = outputStream.ToStreamEventObservable(ReshapingPolicy.None).ToEnumerable().ToArray();
Assert.IsTrue(output.SequenceEqual(compareTo));
}
public static void Main(string[] args)
{
Alpha();
var meths = typeof(int).GetMethods();
var x = new { A = 3, B = 'a' };
var y = EqualsExprForAnonymousType(x.GetType());
var z = TestEquality(Enumerable.Range(0, 5), i => new { X = i, Y = (char)('a' + i), });
z = TestEquality(new int[] { 3, 3, 3 }, i => new { Z = i, A = (char)('a' + i), W = "abc", });
var ii = TestHash(new int[] { 3, 3, 3 }, i => new { Z = i, A = (char)('a' + i), W = "abc", });
ii = TestComparer(new int[] { 3, 3, 3 }, i => new { Z = i, A = (char)('a' + i), W = "abc" });
NativeMethods.AffinitizeThread(0);
Config.ForceRowBasedExecution = true;
int numEvents = 500000;
var leftStream = Observable.Range(0, numEvents / 1000).Select(e => StreamEvent.CreateStart(0, e * 1000)).ToStreamable();
var rightStream = Observable.Range(0, numEvents).Select(e => StreamEvent.CreateStart(0, e)).ToStreamable();
Console.WriteLine("Preloaded");
var sw = new Stopwatch();
sw.Start();
var query = leftStream.Join(rightStream, e => e, e => e, (l, r) => l);
query.ToStreamMessageObservable().ForEachAsync(e => { e.Free(); }).Wait();
sw.Stop();
Console.WriteLine("Throughput: {0}K ev/sec", numEvents / sw.ElapsedMilliseconds);
Console.ReadLine();
}
static void Main(string[] args)
{
// ingress data
var input =
Observable
.Range(0, 100)
.Select(e => new MyStruct {
field1 = e % 10, field2 = e + 0.5, field3 = "blah"
})
.Select(e => StreamEvent.CreateStart(StreamEvent.MinSyncTime, e))
.ToStreamable(OnCompletedPolicy.EndOfStream(), PeriodicPunctuationPolicy.None())
.Cache();
// query
var query =
input.SetProperty().IsConstantDuration(true)
.GroupBy(e => e.field2)
.SelectMany(str => str.Distinct(f => f.field1), (g, c) => c);
// egress data
query
.ToStreamEventObservable()
.Where(e => e.IsData)
.ForEachAsync(e => Console.WriteLine("{0}\t{1}\t{2}", e.SyncTime, e.OtherTime, e.Payload)).Wait();
Console.ReadLine();
}
public void TestAggregateByKey()
{
var input = new[]
{
StreamEvent.CreateStart(0, 100),
StreamEvent.CreateStart(0, 105),
StreamEvent.CreateStart(0, 104),
StreamEvent.CreateStart(0, 200),
StreamEvent.CreateStart(0, 201),
StreamEvent.CreateStart(0, 300),
StreamEvent.CreateStart(0, 302),
StreamEvent.CreateStart(0, 303),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable().SetProperty().IsConstantDuration(true, StreamEvent.InfinitySyncTime);
var output = input
.AggregateByKey(
a => a / 100,
b => b.Count(),
(key, count) => key * 100 + (int)count * 10);
var correct = new[]
{
StreamEvent.CreateStart(0, 130),
StreamEvent.CreateStart(0, 220),
StreamEvent.CreateStart(0, 330),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
public void ReorderTest1Row()
{
using (var modifier = new ConfigModifier().IngressSortingTechnique(SortingTechnique.ImpatienceSort).Modify())
{
var outputList = new List <StreamEvent <int> >();
double disorderFraction = 0.5;
int reorderLatency = 202;
int disorderAmount = 200;
var rand = new Random(2);
var disorderedData =
Enumerable.Range(disorderAmount, 500000).ToList()
.Select(e => StreamEvent.CreateStart(rand.NextDouble() < disorderFraction ? e - rand.Next(0, disorderAmount) : e, 0))
.ToList();
var stream = disorderedData.ToObservable().ToStreamable(DisorderPolicy.Drop(reorderLatency));
stream.ToStreamEventObservable().ForEachAsync(e => { if (e.IsData)
{
outputList.Add(e);
}
}).Wait();
disorderedData.Sort((a, b) => a.SyncTime.CompareTo(b.SyncTime));
Assert.IsTrue(disorderedData.SequenceEqual(outputList));
}
}
public void ExtendTestNegative3()
{
// nothing interesting happens here
var inputList = new[]
{
StreamEvent.CreateStart(1, "A"),
StreamEvent.CreateStart(2, "A"),
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(5, 1, "A"),
StreamEvent.CreateEnd(6, 2, "A"),
StreamEvent.CreateEnd(17, 3, "A"),
StreamEvent.CreateEnd(18, 4, "A"),
};
var compareTo = new[]
{
StreamEvent.CreateStart(3, "A"),
StreamEvent.CreateStart(4, "A"),
StreamEvent.CreateEnd(7, 3, "A"),
StreamEvent.CreateEnd(8, 4, "A"),
END
};
var input = inputList.ToList().ToObservable().ToStreamable().SetProperty().IsIntervalFree(true);
var outputStream = input.ExtendLifetime(-10);
Assert.IsTrue(outputStream.IsEquivalentTo(compareTo));
}
public void PointAtEndTest6()
{
var payload = StructTuple.Create("A", 3);
var inputList = new[]
{
StreamEvent.CreateStart(1, payload),
StreamEvent.CreateEnd(2, 1, payload),
StreamEvent.CreateStart(2, payload),
StreamEvent.CreateEnd(3, 2, payload),
StreamEvent.CreateStart(3, payload),
StreamEvent.CreateEnd(4, 3, payload),
StreamEvent.CreateStart(4, payload),
StreamEvent.CreateEnd(5, 4, payload)
};
var compareTo = new StreamEvent <StructTuple <string, int> >[]
{
StreamEvent.CreatePoint(101, payload),
StreamEvent.CreatePoint(102, payload),
StreamEvent.CreatePoint(103, payload),
StreamEvent.CreatePoint(104, payload),
END2
};
var input = inputList.ToList().ToObservable().ToStreamable()
.AlterEventDuration(100);
var outputStream = input.PointAtEnd();
Assert.IsTrue(outputStream.Select(r => r.Item1).IsEquivalentTo(compareTo.Select(r => r.IsData ? StreamEvent.CreatePoint(r.SyncTime, r.Payload.Item1) : END).ToArray()));
}
static void Main(string[] args)
{
// create the data source
// note: we cannot create more than one of these right now (limitation on the unmanaged side for simplicity)
IObservable <NativeData> source = new NativeObservable();
// create the stream processor
var inputStream = source
.Select(e => StreamEvent.CreateStart(DateTime.Now.Ticks, e))
.ToStreamable(
null,
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.Time((ulong)TimeSpan.FromSeconds(1).Ticks));
// The query
long windowSize = TimeSpan.FromSeconds(2).Ticks;
var query = inputStream
.AlterEventDuration(windowSize)
// we "filter" using our unmanaged matcher
.Where(e => NativeSubscription.MatchData(e.Data))
.Select(e => e.Data);
// Egress results and write to console
query.ToStreamEventObservable().ForEachAsync(e => WriteEvent(e)).Wait();
}
public void TestTopKRobsBug()
{
/*
* Time: 1 2 3 4 5
* Input: 10 |-----------|
* 20 |-----|
* 30 |--|
* 40 |-----|
* 50 |--|
* Output: 20 40 50 10
* 10 30 40
* 20 10
*/
var input = new[]
{
StreamEvent.CreateInterval(1, 5, 10),
StreamEvent.CreateInterval(1, 3, 20),
StreamEvent.CreateInterval(2, 3, 30),
StreamEvent.CreateInterval(2, 4, 40),
StreamEvent.CreateInterval(3, 4, 50),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
}.ToStreamable();
var output = input.Aggregate(a => a.TopK(v => v, 3), b => b.Count(), (a, b) => a.Select(v => v.Payload)).SelectMany(v => v);
var correct = new[]
{
StreamEvent.CreateStart(1, 20),
StreamEvent.CreateEnd(2, 1, 20),
StreamEvent.CreateStart(1, 10),
StreamEvent.CreateEnd(2, 1, 10),
StreamEvent.CreateStart(2, 40),
StreamEvent.CreateEnd(3, 2, 40),
StreamEvent.CreateStart(2, 30),
StreamEvent.CreateEnd(3, 2, 30),
StreamEvent.CreateStart(2, 20),
StreamEvent.CreateEnd(3, 2, 20),
StreamEvent.CreateStart(3, 50),
StreamEvent.CreateEnd(4, 3, 50),
StreamEvent.CreateStart(3, 40),
StreamEvent.CreateEnd(4, 3, 40),
StreamEvent.CreateStart(3, 10),
StreamEvent.CreateEnd(4, 3, 10),
StreamEvent.CreateStart(4, 10),
StreamEvent.CreateEnd(5, 4, 10),
StreamEvent.CreatePunctuation <int>(StreamEvent.InfinitySyncTime)
};
Assert.IsTrue(output.IsEquivalentTo(correct));
}
