public void ShuffleStreamable3Row()
{
var savedUseMultiString = Config.UseMultiString;
var savedScheduler = Config.StreamScheduler;
Config.StreamScheduler = StreamScheduler.OwnedThreads(2);
Config.UseMultiString = true;
var input = Enumerable.Range(0, 100)
.Select(i => new MyData {
field1 = i, field2 = (i % 10).ToString(),
});
var stream = input.ToStatStreamable();
var result = stream
.GroupApply(e => e.field2, str => str.Count(), (g, c) => new StructTuple <string, int> {
Item1 = g.Key, Item2 = (int)c,
})
.ToAtemporalObservable()
.ToEnumerable()
.OrderBy(e => e.Item1)
.ToArray();
var expected = input
.GroupBy(e => e.field2, (k, v) => new StructTuple <string, int> {
Item1 = k, Item2 = v.Count(),
})
.ToArray();
Assert.IsTrue(expected.SequenceEqual(result));
Config.StreamScheduler.Stop();
Config.StreamScheduler = savedScheduler;
Config.UseMultiString = savedUseMultiString;
}
public void ShuffleStreamable1ColumnarSmallBatch()
{
var savedScheduler = Config.StreamScheduler;
Config.StreamScheduler = StreamScheduler.OwnedThreads(2);
var gameInput = new[]
{
StreamEvent.CreateStart(9900, new GameData {
EventType = 0, GameId = 10, UserId = 100
}), // start game
StreamEvent.CreateStart(10000, new GameData {
EventType = 1, GameId = 10, UserId = 100, NumKills = 1
}),
StreamEvent.CreateStart(10020, new GameData {
EventType = 1, GameId = 10, UserId = 100, NumKills = 1
}),
StreamEvent.CreateStart(10025, new GameData {
EventType = 1, GameId = 10, UserId = 100, NumKills = 30
}),
StreamEvent.CreateStart(10030, new GameData {
EventType = 1, GameId = 10, UserId = 100
}), // end game
StreamEvent.CreateStart(10040, new GameData {
EventType = 2, GameId = 10
})
}.ToObservable().ToStreamable();
// clip each game event to end at the time of game completion
var clippedGameInput =
gameInput.Where(e => e.EventType < 2).ClipEventDuration(gameInput.Where(e => e.EventType == 2), e => e.GameId, e => e.GameId);
var result =
clippedGameInput
.GroupApply(
e => new { e.GameId, e.UserId },
str => StreamableInternal.ComputeSignalChangeStream(str.Sum(e => e.NumKills)),
(g, c) => new { g.Key.GameId, g.Key.UserId, FromKills = c.Item1, ToKills = c.Item2 }) // count #kills per {game,user} combination
;
var finalResultSequence = result
.ToStreamEventObservable()
.ToEnumerable()
.ToList();
var finalResult = finalResultSequence.First();
Assert.IsTrue(finalResultSequence.Count() == 1 &&
finalResult.IsPunctuation && finalResult.SyncTime == StreamEvent.InfinitySyncTime);
Config.StreamScheduler.Stop();
Config.StreamScheduler = savedScheduler;
}
static void BenchMarks(int dur, string eng, string test)
{
Config.DataBatchSize = 120000;
Config.FuseFactor = 1;
Config.StreamScheduler = StreamScheduler.OwnedThreads(2);
Config.ForceRowBasedExecution = true;
int duration = dur;
var testcase = test; //normalize, passfilter, fillconst, fillmean, resample, endtoend
var engine = eng;
double time = 0;
const int start = 0;
const int freq = 500;
const int period = 1000 / freq;
const long window = 60000;
const long gap_tol = window;
long count = (duration * freq);
Config.DataGranularity = window;
Func <IStreamable <Empty, Signal> > data = () =>
{
return(new TestObs("test", start, duration, freq)
.Select(e => e.Payload)
.ToTemporalStreamable(e => e.ts, e => e.ts + period)
.Cache()
);
};
Func <IStreamable <Empty, Signal> > abp_data = () =>
{
const int freq = 125;
const int period = 1000 / freq;
return(new TestObs("test", start, duration, freq)
.Select(e => e.Payload)
.ToTemporalStreamable(e => e.ts, e => e.ts + period)
);
};
Func <IStreamable <Empty, Signal> > ecg_data = () =>
{
const int freq = 500;
const int period = 1000 / freq;
return(new TestObs("test", start, duration, freq)
.Select(e => e.Payload)
.ToTemporalStreamable(e => e.ts, e => e.ts + period)
);
};
switch (testcase + "_" + engine)
{
case "normalize_trill":
time = NonFuseTest(data, stream =>
stream
.Normalize(window)
);
break;
case "normalize_lifestream":
time = FuseTest(data, stream =>
stream
.Normalize(period, window))
;
break;
case "passfilter_trill":
time = NonFuseTest(data, stream =>
stream
.BandPassFilter(period, window, 2, 200)
);
break;
case "passfilter_lifestream":
Config.FuseFactor = (int)(window / period);
time = FuseTest(data, stream =>
stream
.BandPassFilter(period, window, 2, 200)
);
break;
case "fillconst_trill":
time = NonFuseTest(data, stream =>
stream
.FillConst(period, gap_tol, 0)
);
break;
case "fillconst_lifestream":
Config.FuseFactor = (int)(window / period);
time = FuseTest(data, stream =>
stream
.FillConst(period, gap_tol, 0)
);
break;
case "fillmean_trill":
time = NonFuseTest(data, stream =>
stream
.FillMean(window, period, gap_tol)
);
break;
case "fillmean_lifestream":
time = FuseTest(data, stream =>
stream
.FillMean(window, period, gap_tol)
);
break;
case "resample_trill":
time = NonFuseTest(data, stream =>
stream
.Resample(period, period / 2)
);
break;
case "resample_lifestream":
Config.FuseFactor = (int)(window / period);
time = FuseTest(data, stream =>
stream
.Resample(period, period / 2, period)
);
break;
case "endtoend_trill":
count = duration * (500 + 125);
time = MultiNonFuseTest(abp_data, ecg_data);
break;
case "endtoend_lifestream":
count = duration * (500 + 125);
time = MultiFuseTest(abp_data, ecg_data);
break;
default:
Console.Error.WriteLine("Unknown benchmark combination {0} on {1}", testcase, engine);
return;
}
Config.StreamScheduler.Stop();
}