public void Group7Columnar() { using (var modifier = new ConfigModifier().UseMultiString(true).Modify()) { var input = Enumerable.Range(0, 100) .Select(i => new MyData { field1 = i, field2 = (i % 10).ToString(), }) ; var stream = input .ToStatStreamable(); var result = stream .GroupApply(e => e.field2, str => str.Count(), (g, c) => new StructTuple <string, int> { Item1 = g.Key, Item2 = (int)c, }) .ToStreamEventObservable().Where(e => e.IsData) .Select(e => e.Payload) .ToEnumerable() .OrderBy(e => e.Item1) .ToArray(); var expected = input .GroupBy(e => e.field2, (k, v) => new StructTuple <string, int> { Item1 = k, Item2 = v.Count(), }) .ToArray() ; Assert.IsTrue(expected.SequenceEqual(result)); } }
public void ReorderTest2ColumnarSmallBatch() { using (var modifier = new ConfigModifier().IngressSortingTechnique(SortingTechnique.PriorityQueue).Modify()) { var outputList = new List <StreamEvent <int> >(); double disorderFraction = 0.5; int reorderLatency = 202; int disorderAmount = 200; var rand = new Random(2); var disorderedData = Enumerable.Range(disorderAmount, 500000).ToList() .Select(e => StreamEvent.CreateStart(rand.NextDouble() < disorderFraction ? e - rand.Next(0, disorderAmount) : e, 0)) .ToList(); var stream = disorderedData.ToObservable().ToStreamable(DisorderPolicy.Drop(reorderLatency)); stream.ToStreamEventObservable().ForEachAsync(e => { if (e.IsData) { outputList.Add(e); } }).Wait(); disorderedData.Sort((a, b) => a.SyncTime.CompareTo(b.SyncTime)); Assert.IsTrue(disorderedData.SequenceEqual(outputList)); } }
internal TriPartitionedOrderedTestsBase( ConfigModifier config, DisorderPolicy disorderPolicy, PeriodicPunctuationPolicy punctuationPolicy, PeriodicLowWatermarkPolicy lowWatermarkPolicy) : base(config) { this.disorderPolicy = disorderPolicy; this.punctuationPolicy = punctuationPolicy; this.lowWatermarkPolicy = lowWatermarkPolicy; }
public void ShuffleStreamable1ColumnarSmallBatch() { using (var modifier = new ConfigModifier().DefaultScheduler(StreamScheduler.OwnedThreads(2)).Modify()) { var gameInput = new[] { StreamEvent.CreateStart(9900, new GameData { EventType = 0, GameId = 10, UserId = 100 }), // start game StreamEvent.CreateStart(10000, new GameData { EventType = 1, GameId = 10, UserId = 100, NumKills = 1 }), StreamEvent.CreateStart(10020, new GameData { EventType = 1, GameId = 10, UserId = 100, NumKills = 1 }), StreamEvent.CreateStart(10025, new GameData { EventType = 1, GameId = 10, UserId = 100, NumKills = 30 }), StreamEvent.CreateStart(10030, new GameData { EventType = 1, GameId = 10, UserId = 100 }), // end game StreamEvent.CreateStart(10040, new GameData { EventType = 2, GameId = 10 }) }.ToObservable().ToStreamable(); // clip each game event to end at the time of game completion var clippedGameInput = gameInput.Where(e => e.EventType < 2).ClipEventDuration(gameInput.Where(e => e.EventType == 2), e => e.GameId, e => e.GameId); var result = clippedGameInput .GroupApply( e => new { e.GameId, e.UserId }, str => StreamableInternal.ComputeSignalChangeStream(str.Sum(e => e.NumKills)), (g, c) => new { g.Key.GameId, g.Key.UserId, FromKills = c.Item1, ToKills = c.Item2 }) // count #kills per {game,user} combination ; var finalResultSequence = result .ToStreamEventObservable() .ToEnumerable() .ToList(); var finalResult = finalResultSequence.First(); Assert.IsTrue(finalResultSequence.Count() == 1 && finalResult.IsPunctuation && finalResult.SyncTime == StreamEvent.InfinitySyncTime); } }
private static void EquiJoinStreamable(bool columnar, bool fixedInterval) { using (var modifier = new ConfigModifier() .ForceRowBasedExecution(!columnar) .DontFallBackToRowBasedExecution(columnar) .Modify()) { JoinIntervalsTest(fixedInterval); JoinPointsTest(fixedInterval); if (!fixedInterval) { JoinEdgeIntervalTest(); JoinEdgesTest(); PartitionedStartEdgeJoinTest(); } } }
public void SorterDequeueUntillRow() { using (var modifier = new ConfigModifier().IngressSortingTechnique(SortingTechnique.ImpatienceSort).Modify()) { var input = new List <Tuple <int, int> >() { Tuple.Create(0, 1), Tuple.Create(1, 1), Tuple.Create(0, 3), Tuple.Create(0, 5), Tuple.Create(1, 3), Tuple.Create(1, 20), Tuple.Create(0, 6), Tuple.Create(0, 7), Tuple.Create(0, 8), Tuple.Create(1, 2), Tuple.Create(0, 2), Tuple.Create(1, 30), Tuple.Create(0, 13) }; var expectedoutput = new List <Tuple <int, int> >() { Tuple.Create(1, 1), Tuple.Create(1, 3), Tuple.Create(1, 20), Tuple.Create(0, 1), Tuple.Create(0, 2), Tuple.Create(0, 3), Tuple.Create(0, 5), Tuple.Create(0, 6), Tuple.Create(0, 7), Tuple.Create(0, 8), Tuple.Create(0, 13), Tuple.Create(1, 30) }; var prog = input.Select(x => PartitionedStreamEvent.CreateStart(x.Item1, x.Item2, x.Item2)).ToObservable() .ToStreamable(DisorderPolicy.Drop(10)).ToStreamEventObservable(); var outevents = prog.ToEnumerable().ToList(); var output = outevents.Where(o => o.IsData).ToList(); var success = output.SequenceEqual(expectedoutput.Select(t => PartitionedStreamEvent.CreateStart(t.Item1, t.Item2, t.Item2))); Assert.IsTrue(success); } }
public void ShuffleStreamable3Row() { using (var modifier = new ConfigModifier().UseMultiString(true) //// TODO: OwnedThreads(2) causes intermittent failures .DefaultScheduler(StreamScheduler.OwnedThreads(2)) .Modify()) { var input = Enumerable.Range(0, 100) .Select(i => new MyData { field1 = i, field2 = (i % 10).ToString(), }); var stream = input.ToStatStreamable(); var result = stream .GroupApply(e => e.field2, str => str.Count(), (g, c) => new StructTuple <string, int> { Item1 = g.Key, Item2 = (int)c, }) .ToAtemporalObservable() .ToEnumerable() .OrderBy(e => e.Item1) .ToArray(); var expected = input .GroupBy(e => e.field2, (k, v) => new StructTuple <string, int> { Item1 = k, Item2 = v.Count(), }) .ToArray(); if (!expected.SequenceEqual(result)) { string message = $"Did not receive expected results! Expected:{System.Environment.NewLine}"; foreach (var expectedEvent in expected) { message += $"{expectedEvent}{System.Environment.NewLine}"; } message += $"{System.Environment.NewLine}Actual:{System.Environment.NewLine}"; foreach (var actualEvent in result) { message += $"{actualEvent}{System.Environment.NewLine}"; } Assert.IsTrue(false, message); } } }
public void Group0ColumnarSmallBatch() { using (var modifier = new ConfigModifier().MapArity(1).ReduceArity(1).Modify()) { // Simple pass through var input = Enumerable.Range(0, 20); var cached = input .ToObservable(); var cachedStr = cached .ToTemporalStreamable(s => 0, s => StreamEvent.InfinitySyncTime); var streamResult = cachedStr .GroupApply(i => i % 2, g => g) .ToPayloadEnumerable() ; var a = streamResult.ToArray(); Assert.IsTrue(input.SequenceEqual(a)); } }
public void ShuffleStreamable4ColumnarSmallBatch() { using (var modifier = new ConfigModifier().UseMultiString(true).Modify()) { var input = Enumerable.Range(0, 100) .Select(i => new MyData { field1 = i, field2 = (i % 10).ToString(), }); var stream = input.ToStatStreamable(); var result = stream .Shard(2) .ReKey(e => e.field2) .ReDistribute() .Query(str => str.Count()) .SelectKey((g, c) => new StructTuple <string, int> { Item1 = g, Item2 = (int)c, }) .Unshuffle() .Unshard() .ToAtemporalObservable() .ToEnumerable() .OrderBy(e => e.Item1) .ToArray(); var ct = result.Length; var expected = input .GroupBy(e => e.field2, (k, v) => new StructTuple <string, int> { Item1 = k, Item2 = v.Count(), }) .ToArray(); Assert.IsTrue(expected.SequenceEqual(result)); } // TODO: this test has an intermittent memory leak only in the lab. Temporarily disable until this can be debugged. MemoryManager.Free(true); }
internal TestWithConfigSettingsAndMemoryLeakDetection(ConfigModifier modifier) { this.modifier = modifier; }
internal TestWithConfigSettingsWithoutMemoryLeakDetection(ConfigModifier modifier) => this.modifier = modifier;