public void Group7Columnar()
{
using (var modifier = new ConfigModifier().UseMultiString(true).Modify())
{
var input = Enumerable.Range(0, 100)
.Select(i => new MyData {
field1 = i, field2 = (i % 10).ToString(),
})
;
var stream = input
.ToStatStreamable();
var result = stream
.GroupApply(e => e.field2, str => str.Count(), (g, c) => new StructTuple <string, int> {
Item1 = g.Key, Item2 = (int)c,
})
.ToStreamEventObservable().Where(e => e.IsData)
.Select(e => e.Payload)
.ToEnumerable()
.OrderBy(e => e.Item1)
.ToArray();
var expected = input
.GroupBy(e => e.field2, (k, v) => new StructTuple <string, int> {
Item1 = k, Item2 = v.Count(),
})
.ToArray()
;
Assert.IsTrue(expected.SequenceEqual(result));
}
}
public void ReorderTest2ColumnarSmallBatch()
{
using (var modifier = new ConfigModifier().IngressSortingTechnique(SortingTechnique.PriorityQueue).Modify())
{
var outputList = new List <StreamEvent <int> >();
double disorderFraction = 0.5;
int reorderLatency = 202;
int disorderAmount = 200;
var rand = new Random(2);
var disorderedData =
Enumerable.Range(disorderAmount, 500000).ToList()
.Select(e => StreamEvent.CreateStart(rand.NextDouble() < disorderFraction ? e - rand.Next(0, disorderAmount) : e, 0))
.ToList();
var stream = disorderedData.ToObservable().ToStreamable(DisorderPolicy.Drop(reorderLatency));
stream.ToStreamEventObservable().ForEachAsync(e => { if (e.IsData)
{
outputList.Add(e);
}
}).Wait();
disorderedData.Sort((a, b) => a.SyncTime.CompareTo(b.SyncTime));
Assert.IsTrue(disorderedData.SequenceEqual(outputList));
}
}
internal TriPartitionedOrderedTestsBase(
ConfigModifier config,
DisorderPolicy disorderPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
PeriodicLowWatermarkPolicy lowWatermarkPolicy) : base(config)
{
this.disorderPolicy = disorderPolicy;
this.punctuationPolicy = punctuationPolicy;
this.lowWatermarkPolicy = lowWatermarkPolicy;
}
public void ShuffleStreamable1ColumnarSmallBatch()
{
using (var modifier = new ConfigModifier().DefaultScheduler(StreamScheduler.OwnedThreads(2)).Modify())
{
var gameInput = new[]
{
StreamEvent.CreateStart(9900, new GameData {
EventType = 0, GameId = 10, UserId = 100
}), // start game
StreamEvent.CreateStart(10000, new GameData {
EventType = 1, GameId = 10, UserId = 100, NumKills = 1
}),
StreamEvent.CreateStart(10020, new GameData {
EventType = 1, GameId = 10, UserId = 100, NumKills = 1
}),
StreamEvent.CreateStart(10025, new GameData {
EventType = 1, GameId = 10, UserId = 100, NumKills = 30
}),
StreamEvent.CreateStart(10030, new GameData {
EventType = 1, GameId = 10, UserId = 100
}), // end game
StreamEvent.CreateStart(10040, new GameData {
EventType = 2, GameId = 10
})
}.ToObservable().ToStreamable();
// clip each game event to end at the time of game completion
var clippedGameInput =
gameInput.Where(e => e.EventType < 2).ClipEventDuration(gameInput.Where(e => e.EventType == 2), e => e.GameId, e => e.GameId);
var result =
clippedGameInput
.GroupApply(
e => new { e.GameId, e.UserId },
str => StreamableInternal.ComputeSignalChangeStream(str.Sum(e => e.NumKills)),
(g, c) => new { g.Key.GameId, g.Key.UserId, FromKills = c.Item1, ToKills = c.Item2 }) // count #kills per {game,user} combination
;
var finalResultSequence = result
.ToStreamEventObservable()
.ToEnumerable()
.ToList();
var finalResult = finalResultSequence.First();
Assert.IsTrue(finalResultSequence.Count() == 1 &&
finalResult.IsPunctuation && finalResult.SyncTime == StreamEvent.InfinitySyncTime);
}
}
private static void EquiJoinStreamable(bool columnar, bool fixedInterval)
{
using (var modifier = new ConfigModifier()
.ForceRowBasedExecution(!columnar)
.DontFallBackToRowBasedExecution(columnar)
.Modify())
{
JoinIntervalsTest(fixedInterval);
JoinPointsTest(fixedInterval);
if (!fixedInterval)
{
JoinEdgeIntervalTest();
JoinEdgesTest();
PartitionedStartEdgeJoinTest();
}
}
}
public void SorterDequeueUntillRow()
{
using (var modifier = new ConfigModifier().IngressSortingTechnique(SortingTechnique.ImpatienceSort).Modify())
{
var input = new List <Tuple <int, int> >()
{
Tuple.Create(0, 1),
Tuple.Create(1, 1),
Tuple.Create(0, 3),
Tuple.Create(0, 5),
Tuple.Create(1, 3),
Tuple.Create(1, 20),
Tuple.Create(0, 6),
Tuple.Create(0, 7),
Tuple.Create(0, 8),
Tuple.Create(1, 2),
Tuple.Create(0, 2),
Tuple.Create(1, 30),
Tuple.Create(0, 13)
};
var expectedoutput = new List <Tuple <int, int> >()
{
Tuple.Create(1, 1),
Tuple.Create(1, 3),
Tuple.Create(1, 20),
Tuple.Create(0, 1),
Tuple.Create(0, 2),
Tuple.Create(0, 3),
Tuple.Create(0, 5),
Tuple.Create(0, 6),
Tuple.Create(0, 7),
Tuple.Create(0, 8),
Tuple.Create(0, 13),
Tuple.Create(1, 30)
};
var prog = input.Select(x => PartitionedStreamEvent.CreateStart(x.Item1, x.Item2, x.Item2)).ToObservable()
.ToStreamable(DisorderPolicy.Drop(10)).ToStreamEventObservable();
var outevents = prog.ToEnumerable().ToList();
var output = outevents.Where(o => o.IsData).ToList();
var success = output.SequenceEqual(expectedoutput.Select(t => PartitionedStreamEvent.CreateStart(t.Item1, t.Item2, t.Item2)));
Assert.IsTrue(success);
}
}
public void ShuffleStreamable3Row()
{
using (var modifier = new ConfigModifier().UseMultiString(true)
//// TODO: OwnedThreads(2) causes intermittent failures .DefaultScheduler(StreamScheduler.OwnedThreads(2))
.Modify())
{
var input = Enumerable.Range(0, 100)
.Select(i => new MyData {
field1 = i, field2 = (i % 10).ToString(),
});
var stream = input.ToStatStreamable();
var result = stream
.GroupApply(e => e.field2, str => str.Count(), (g, c) => new StructTuple <string, int> {
Item1 = g.Key, Item2 = (int)c,
})
.ToAtemporalObservable()
.ToEnumerable()
.OrderBy(e => e.Item1)
.ToArray();
var expected = input
.GroupBy(e => e.field2, (k, v) => new StructTuple <string, int> {
Item1 = k, Item2 = v.Count(),
})
.ToArray();
if (!expected.SequenceEqual(result))
{
string message = $"Did not receive expected results! Expected:{System.Environment.NewLine}";
foreach (var expectedEvent in expected)
{
message += $"{expectedEvent}{System.Environment.NewLine}";
}
message += $"{System.Environment.NewLine}Actual:{System.Environment.NewLine}";
foreach (var actualEvent in result)
{
message += $"{actualEvent}{System.Environment.NewLine}";
}
Assert.IsTrue(false, message);
}
}
}
public void Group0ColumnarSmallBatch()
{
using (var modifier = new ConfigModifier().MapArity(1).ReduceArity(1).Modify())
{
// Simple pass through
var input = Enumerable.Range(0, 20);
var cached = input
.ToObservable();
var cachedStr = cached
.ToTemporalStreamable(s => 0, s => StreamEvent.InfinitySyncTime);
var streamResult = cachedStr
.GroupApply(i => i % 2, g => g)
.ToPayloadEnumerable()
;
var a = streamResult.ToArray();
Assert.IsTrue(input.SequenceEqual(a));
}
}
public void ShuffleStreamable4ColumnarSmallBatch()
{
using (var modifier = new ConfigModifier().UseMultiString(true).Modify())
{
var input = Enumerable.Range(0, 100)
.Select(i => new MyData {
field1 = i, field2 = (i % 10).ToString(),
});
var stream = input.ToStatStreamable();
var result = stream
.Shard(2)
.ReKey(e => e.field2)
.ReDistribute()
.Query(str => str.Count())
.SelectKey((g, c) => new StructTuple <string, int> {
Item1 = g, Item2 = (int)c,
})
.Unshuffle()
.Unshard()
.ToAtemporalObservable()
.ToEnumerable()
.OrderBy(e => e.Item1)
.ToArray();
var ct = result.Length;
var expected = input
.GroupBy(e => e.field2, (k, v) => new StructTuple <string, int> {
Item1 = k, Item2 = v.Count(),
})
.ToArray();
Assert.IsTrue(expected.SequenceEqual(result));
}
// TODO: this test has an intermittent memory leak only in the lab. Temporarily disable until this can be debugged.
MemoryManager.Free(true);
}
internal TestWithConfigSettingsAndMemoryLeakDetection(ConfigModifier modifier)
{
this.modifier = modifier;
}
internal TestWithConfigSettingsWithoutMemoryLeakDetection(ConfigModifier modifier) => this.modifier = modifier;
