private static void TryStatementCreateSendAndStop(
RegressionEnvironment env,
int numThreads,
StmtMgmtCallablePair[] statements,
int numRepeats)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtMgmt)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtMgmtCallable(env.Runtime, statements, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
var statementDigest = new StringBuilder();
for (var i = 0; i < statements.Length; i++) {
statementDigest.Append(statements[i].Epl);
}
SupportCompileDeployUtil.AssertFutures(future);
}
private static void TrySend(
RegressionEnvironment env,
EPStatement stmtWindow,
int numThreads,
int numRepeats)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowPriority)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtNamedWindowPriorityCallable(i, env.Runtime, numRepeats);
future[i] = threadPool.Submit(callable);
}
SupportCompileDeployUtil.AssertFutures(future);
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
var events = EPAssertionUtil.EnumeratorToArray(stmtWindow.GetEnumerator());
Assert.AreEqual(numThreads * numRepeats, events.Length);
for (var i = 0; i < events.Length; i++) {
var valueC1 = (string) events[i].Get("c1");
Assert.AreEqual("y", valueC1);
}
}
private static void TryCount(
RegressionEnvironment env,
int numThreads,
int numMessages,
string epl,
GeneratorEnumeratorCallback generatorEnumeratorCallback)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtStatelessEnummethod)).ThreadFactory);
env.CompileDeploy(epl);
var listener = new SupportMTUpdateListener();
env.Statement("s0").AddListener(listener);
var future = new IFuture <bool> [numThreads];
for (var i = 0; i < numThreads; i++)
{
future[i] = threadPool.Submit(
new SendEventCallable(
i,
env.Runtime,
new GeneratorEnumerator(numMessages, generatorEnumeratorCallback)));
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
Assert.AreEqual(numMessages * numThreads, listener.GetNewDataListFlattened().Length);
}
public void Run(Configuration configuration)
{
idCounter = new AtomicLong(0);
executorService = Executors.NewCachedThreadPool();
noActionUpdateListener = new NoActionUpdateListener();
configuration.Runtime.Threading.IsInternalTimerEnabled = true;
configuration.Common.AddEventType(typeof(SupportBean));
configuration.Runtime.Threading.InsertIntoDispatchLocking = Locking.SUSPEND;
var runtime = EPRuntimeProvider.GetRuntime(GetType().Name, configuration);
runtime.Initialize();
epRuntime = runtime.EventService;
var path = new RegressionPath();
var epl = "insert into Stream1 select count(*) as cnt from SupportBean#time(7 sec)";
var compiled = SupportCompileDeployUtil.Compile(epl, configuration, path);
path.Add(compiled);
SupportCompileDeployUtil.Deploy(compiled, runtime);
epl = epl + " output every 10 seconds";
compiled = SupportCompileDeployUtil.Compile(epl, configuration, path);
SupportCompileDeployUtil.DeployAddListener(compiled, "insert", noActionUpdateListener, runtime);
var sendTickEventRunnable = new SendEventRunnable(this, 10000);
Start(sendTickEventRunnable, 4);
// Adjust here for long-running test
SupportCompileDeployUtil.ThreadSleep(3000);
sendTickEventRunnable.Shutdown = true;
executorService.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(executorService, 1, TimeUnit.SECONDS);
runtime.Destroy();
}
private static void TrySend(
RegressionEnvironment env,
int numThreads,
int numRepeats)
{
var compiled = env.Compile("@Name('upd') update istream SupportBean set TheString='a'");
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadUpdate)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtUpdateSendCallable(i, env.Runtime, numRepeats);
future[i] = threadPool.Submit(callable);
}
for (var i = 0; i < 50; i++) {
env.Deploy(compiled);
SupportCompileDeployUtil.ThreadSleep(10);
env.UndeployModuleContaining("upd");
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 5, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
}
private static void TryThreadSafetyHistoricalJoin(
RegressionEnvironment env,
int numThreads,
int numRepeats)
{
var listener = new MyListener();
env.Statement("select").AddListener(listener);
var events = new IList<object>[numThreads];
for (var threadNum = 0; threadNum < numThreads; threadNum++) {
events[threadNum] = new List<object>();
for (var eventNum = 0; eventNum < numRepeats; eventNum++) {
// range: 1 to 1000
var partition = eventNum + 1;
events[threadNum].Add(new SupportBean(new int?(partition).ToString(), 0));
}
}
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadContextDBAccess)).ThreadFactory);
var futures = new IFuture<bool>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new SendEventCallable(i, env.Runtime, events[i].GetEnumerator());
futures[i] = threadPool.Submit(callable);
}
SupportCompileDeployUtil.AssertFutures(futures);
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
Assert.AreEqual(numRepeats * numThreads, listener.Count);
}
private static void TrySend(
RegressionEnvironment env,
int numThreads,
int numEventsPerThread,
bool indexShare)
{
// setup statements
var path = new RegressionPath();
var schemas = "create schema UpdateEvent as (uekey string, ueint int);\n" +
"create schema WindowSchema as (wskey string, wsint int);\n";
env.CompileDeployWBusPublicType(schemas, path);
var createEpl = "@Name('namedWindow') create window MyWindow#keepall as WindowSchema";
if (indexShare) {
createEpl = "@Hint('enable_window_subquery_indexshare') " + createEpl;
}
env.CompileDeploy(createEpl, path);
env.CompileDeploy("create index ABC on MyWindow(wskey)", path);
env.CompileDeploy(
"on UpdateEvent mue merge MyWindow mw " +
"where uekey = wskey and ueint = wsint " +
"when not matched then insert select uekey as wskey, ueint as wsint " +
"when matched then delete",
path);
// note: here all threads use the same string key to insert/delete and different values for the int
env.CompileDeploy(
"@Name('target') select (select intListAgg(wsint) from MyWindow mw where wskey = sb.TheString) as val from SupportBean sb",
path);
// execute
var executor = Executors.NewMultiThreadedExecutor(numThreads);
// new SupportThreadFactory(typeof(MultithreadStmtNamedWindowSubqueryAgg)).ThreadFactory)
var futures = new List<IFuture<bool?>>();
for (var i = 0; i < numThreads; i++) {
futures.Add(executor.Submit(
new StmtNamedWindowSubqueryAggCallable(
i,
env.Runtime,
numEventsPerThread,
env.Statement("target"))));
}
// Give the futures 10 seconds to complete the futures...
futures.AsParallel().ForAll(future => future.Wait(TimeSpan.FromSeconds(10)));
executor.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(executor, TimeSpan.FromSeconds(10));
SupportCompileDeployUtil.AssertFutures(futures);
var events = EPAssertionUtil.EnumeratorToArray(env.Statement("namedWindow").GetEnumerator());
Assert.AreEqual(0, events.Length);
env.UndeployAll();
}
private static void TrySend(
RegressionEnvironment env,
int numThreads,
int numRepeats)
{
// set time to 0
env.AdvanceTime(0);
var listener = new SupportMTUpdateListener();
env.CompileDeploy(
"@Name('s0') select irstream IntPrimitive, TheString as key from SupportBean#time(1 sec)");
env.Statement("s0").AddListener(listener);
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtTimeWindow)).ThreadFactory);
var futures = new List<IFuture<bool>>();
for (var i = 0; i < numThreads; i++) {
var callable = new SendEventCallable(i, env.Runtime, new GeneratorEnumerator(numRepeats));
futures.Add(threadPool.Submit(callable));
}
// Advance time window every 100 milliseconds for 1 second
for (var i = 0; i < 10; i++) {
env.AdvanceTime(i * 1000);
SupportCompileDeployUtil.ThreadSleep(100);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(futures);
// set time to a large value
env.AdvanceTime(10000000000L);
// Assert results
var totalExpected = numThreads * numRepeats;
// assert new data
var resultNewData = listener.GetNewDataListFlattened();
Assert.AreEqual(totalExpected, resultNewData.Length);
var resultsNewData = SortPerIntKey(resultNewData);
AssertResult(numRepeats, numThreads, resultsNewData);
// assert old data
var resultOldData = listener.GetOldDataListFlattened();
Assert.AreEqual(totalExpected, resultOldData.Length);
var resultsOldData = SortPerIntKey(resultOldData);
AssertResult(numRepeats, numThreads, resultsOldData);
env.UndeployAll();
}
private static void TryPattern(
RegressionEnvironment env,
string pattern,
int numThreads,
int numEvents)
{
var sendLock = new object();
var executor = Executors.NewMultiThreadedExecutor(numThreads);
// new SupportThreadFactory(typeof(MultithreadStmtPattern)).ThreadFactory);
var futures = new List<IFuture<object>>();
var callables = new SendEventWaitCallable[numThreads];
for (var i = 0; i < numThreads; i++) {
callables[i] = new SendEventWaitCallable(i, env.Runtime, sendLock, new GeneratorEnumerator(numEvents));
futures.Add(executor.Submit(callables[i]));
}
var listener = new SupportMTUpdateListener[numEvents];
var epl = "select * from pattern[" + pattern + "]";
var compiled = env.Compile(epl);
for (var i = 0; i < numEvents; i++) {
var stmtName = "p" + i;
env.Deploy(compiled, new DeploymentOptions().WithStatementNameRuntime(ctx => stmtName));
var stmt = env.Statement(stmtName);
listener[i] = new SupportMTUpdateListener();
stmt.AddListener(listener[i]);
lock (sendLock) {
Monitor.PulseAll(sendLock);
}
}
foreach (var callable in callables) {
callable.Shutdown();
}
lock (sendLock) {
Monitor.PulseAll(sendLock);
}
executor.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(executor, TimeSpan.FromSeconds(10));
for (var i = 0; i < numEvents; i++) {
var theEventReceived = listener[i].AssertOneGetNewAndReset();
Assert.NotNull(theEventReceived);
var theEventValue = theEventReceived.Get("a");
Assert.NotNull(theEventValue);
Assert.IsInstanceOf<SupportBean>(theEventValue);
}
env.UndeployAll();
}
public void Run(RegressionEnvironment env)
{
var epl = "create schema ScoreCycle (userId string, keyword string, ProductId string, score long);\n" +
"\n" +
"create schema UserKeywordTotalStream (userId string, keyword string, sumScore long);\n" +
"\n" +
"create context HashByUserCtx as\n" +
"coalesce by consistent_hash_crc32(userId) from ScoreCycle,\n" +
"consistent_hash_crc32(userId) from UserKeywordTotalStream \n" +
"granularity 10000000;\n" +
"\n" +
"context HashByUserCtx create window ScoreCycleWindow#unique(userId, keyword, ProductId) as ScoreCycle;\n" +
"\n" +
"context HashByUserCtx insert into ScoreCycleWindow select * from ScoreCycle;\n" +
"\n" +
"@Name('Select') context HashByUserCtx insert into UserKeywordTotalStream\n" +
"select userId, keyword, sum(score) as sumScore from ScoreCycleWindow group by userId, keyword;";
env.CompileDeployWBusPublicType(epl, new RegressionPath());
var listener = new MyUpdateListener();
env.Statement("Select").AddListener(listener);
IList<IDictionary<string, object>> sendsT1 = new List<IDictionary<string, object>>();
sendsT1.Add(MakeEvent("A", "house", "P0", 1));
sendsT1.Add(MakeEvent("B", "house", "P0", 2));
IList<IDictionary<string, object>> sendsT2 = new List<IDictionary<string, object>>();
sendsT2.Add(MakeEvent("B", "house", "P0", 3));
sendsT1.Add(MakeEvent("A", "house", "P0", 4));
var threadPool = Executors.NewFixedThreadPool(
2,
new SupportThreadFactory(typeof(MultithreadContextUnique)).ThreadFactory);
var runnableOne = new SendEventRunnable(env.Runtime, sendsT1, "ScoreCycle");
var runnableTwo = new SendEventRunnable(env.Runtime, sendsT2, "ScoreCycle");
threadPool.Submit(runnableOne.Run);
threadPool.Submit(runnableTwo.Run);
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 1, TimeUnit.SECONDS);
Assert.IsNull(runnableOne.LastException);
Assert.IsNull(runnableTwo.LastException);
// compare
var received = listener.Received;
foreach (var item in received) {
Console.Out.WriteLine(item);
}
Assert.AreEqual(4, received.Count);
env.UndeployAll();
}
private static void TrySend(
int numThreads,
int numEvents,
bool isPreserveOrder,
Locking locking,
Configuration configuration)
{
configuration.Runtime.Threading.IsListenerDispatchPreserveOrder = isPreserveOrder;
configuration.Runtime.Threading.ListenerDispatchLocking = locking;
configuration.Common.AddEventType(typeof(SupportBean));
var runtime = EPRuntimeProvider.GetRuntime(typeof(MultithreadDeterminismListener).Name, configuration);
runtime.Initialize();
// setup statements
var deployed = SupportCompileDeployUtil.CompileDeploy(
"@Name('s0') select count(*) as cnt from SupportBean",
runtime,
configuration);
var listener = new SupportMTUpdateListener();
deployed.Statements[0].AddListener(listener);
// execute
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadDeterminismListener)).ThreadFactory);
var future = new IFuture<bool>[numThreads];
for (var i = 0; i < numThreads; i++) {
future[i] = threadPool.Submit(new SendEventCallable(i, runtime, new GeneratorEnumerator(numEvents)));
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
var events = listener.GetNewDataListFlattened();
var result = new long[events.Length];
for (var i = 0; i < events.Length; i++) {
result[i] = events[i].Get("cnt").AsInt64();
}
//log.info(".trySend result=" + Arrays.toString(result));
// assert result
Assert.AreEqual(numEvents * numThreads, events.Length);
for (var i = 0; i < numEvents * numThreads; i++) {
Assert.AreEqual(result[i], (long) i + 1);
}
runtime.Destroy();
}
private static void TryPerformanceDispatch(
RegressionEnvironment env,
int numThreads,
int numRepeats)
{
var listener = new MyListener();
env.Statement("select").AddListener(listener);
var random = new Random();
var events = new IList<object>[numThreads];
var eventId = 0;
for (var threadNum = 0; threadNum < numThreads; threadNum++) {
events[threadNum] = new List<object>();
for (var eventNum = 0; eventNum < numRepeats; eventNum++) {
// range: 1 to 1000
var partition = random.Next(0, 50);
eventId++;
events[threadNum].Add(new SupportBean(new int?(partition).ToString(), eventId));
}
}
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadContextPartitioned)).ThreadFactory);
var futures = new IFuture<bool>[numThreads];
var startTime = PerformanceObserver.MilliTime;
for (var i = 0; i < numThreads; i++) {
var callable = new SendEventCallable(i, env.Runtime, events[i].GetEnumerator());
futures[i] = threadPool.Submit(callable);
}
SupportCompileDeployUtil.AssertFutures(futures);
var delta = PerformanceObserver.MilliTime - startTime;
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
// print those events not received
foreach (var eventList in events) {
foreach (var @event in eventList) {
if (!listener.Beans.Contains(@event)) {
log.Info("Expected event was not received, event " + @event);
}
}
}
Assert.AreEqual(numRepeats * numThreads, listener.Beans.Count);
Assert.That(delta, Is.LessThan(500));
}
private static void TryListener(
RegressionEnvironment env,
int numThreads,
int numRoutes)
{
env.CompileDeploy("@Name('trigger') select * from SupportBean");
env.CompileDeploy("@Name('s0') select * from SupportMarketDataBean");
var listener = new SupportMTUpdateListener();
env.Statement("s0").AddListener(listener);
// Set of events routed by each listener
ISet<SupportMarketDataBean> routed = new HashSet<SupportMarketDataBean>().AsSyncSet();
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtListenerRoute)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtListenerCreateStmtCallable(
i,
env.Runtime,
env.Statement("trigger"),
numRoutes,
routed);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// assert
var results = listener.GetNewDataListFlattened();
Assert.IsTrue(results.Length >= numThreads * numRoutes);
foreach (var routedEvent in routed) {
var found = false;
for (var i = 0; i < results.Length; i++) {
if (results[i].Underlying == routedEvent) {
found = true;
}
}
Assert.IsTrue(found);
}
env.UndeployAll();
}
private static void TrySend(
RegressionEnvironment env,
SupportMTUpdateListener listenerWindow,
SupportMTUpdateListener listenerConsumer,
int numThreads,
int numRepeats)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowDelete)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtNamedWindowDeleteCallable(Convert.ToString(i), env.Runtime, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
// compute list of expected
IList<string> expectedIdsList = new List<string>();
for (var i = 0; i < numThreads; i++) {
try {
expectedIdsList.AddAll((IList<string>) future[i].Get());
}
catch (Exception t) {
throw new EPException(t);
}
}
var expectedIds = expectedIdsList.ToArray();
Assert.AreEqual(2 * numThreads * numRepeats, listenerWindow.NewDataList.Count); // old and new each
Assert.AreEqual(2 * numThreads * numRepeats, listenerConsumer.NewDataList.Count); // old and new each
// compute list of received
var newEvents = listenerWindow.GetNewDataListFlattened();
var receivedIds = new string[newEvents.Length];
for (var i = 0; i < newEvents.Length; i++) {
receivedIds[i] = (string) newEvents[i].Get("TheString");
}
Assert.AreEqual(receivedIds.Length, expectedIds.Length);
Array.Sort(receivedIds);
Array.Sort(expectedIds);
CompatExtensions.DeepEqualsWithType(expectedIds, receivedIds);
}
private static void TrySend(
RegressionEnvironment env,
SupportMTUpdateListener listener,
int numThreads,
int numRepeats)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtInsertInto)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtInsertIntoCallable(Convert.ToString(i), env.Runtime, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// Assert results
var totalExpected = numThreads * numRepeats * 2;
var result = listener.GetNewDataListFlattened();
Assert.AreEqual(totalExpected, result.Length);
IDictionary<long, ICollection<string>> results = new Dictionary<long, ICollection<string>>();
foreach (var theEvent in result) {
var count = theEvent.Get("mycount").AsInt64();
var key = (string) theEvent.Get("key");
var entries = results.Get(count);
if (entries == null) {
entries = new HashSet<string>();
results.Put(count, entries);
}
entries.Add(key);
}
Assert.AreEqual(numRepeats, results.Count);
foreach (var value in results.Values) {
Assert.AreEqual(2 * numThreads, value.Count);
for (var i = 0; i < numThreads; i++) {
Assert.IsTrue(value.Contains("E1_" + i));
Assert.IsTrue(value.Contains("E2_" + i));
}
}
listener.Reset();
}
private static void TrySendContextCountSimple(
RegressionEnvironment env,
int numThreads,
int numRepeats)
{
var listener = new SupportMTUpdateListener();
env.Statement("select").AddListener(listener);
IList<object>[] eventsPerThread = new List<object>[numThreads];
for (var t = 0; t < numThreads; t++) {
eventsPerThread[t] = new List<object>();
for (var i = 0; i < numRepeats; i++) {
eventsPerThread[t].Add(new SupportBean_S0(-1, "E" + i, i + "_" + t));
}
}
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadContextCountSimple)).ThreadFactory);
var future = new IFuture<bool>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new SendEventCallable(i, env.Runtime, eventsPerThread[i].GetEnumerator());
future[i] = threadPool.Submit(callable);
}
SupportCompileDeployUtil.ThreadSleep(2000);
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
var result = listener.GetNewDataListFlattened();
ISet<string> received = new HashSet<string>();
foreach (var @event in result) {
var key = (string) @event.Get("P01");
if (received.Contains(key)) {
Assert.Fail("key " + key + " received multiple times");
}
received.Add(key);
}
if (received.Count != numRepeats * numThreads) {
log.Info("Received are " + received.Count + " entries");
Assert.Fail();
}
}
private void tryCount(
RegressionEnvironment env,
int numThreads,
int numMessages,
string epl,
GeneratorEnumeratorCallback generatorEnumeratorCallback)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtFilter)).ThreadFactory);
env.CompileDeploy(epl);
var listener = new MTListener("mycount");
env.Statement("s0").AddListener(listener);
var future = new IFuture <bool> [numThreads];
for (var i = 0; i < numThreads; i++)
{
future[i] = threadPool.Submit(
new SendEventCallable(
i,
env.Runtime,
new GeneratorEnumerator(numMessages, generatorEnumeratorCallback)));
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// verify results
Assert.AreEqual(numMessages * numThreads, listener.Values.Count);
var result = new SortedSet <int>();
foreach (var row in listener.Values)
{
result.Add(row.AsInt32());
}
Assert.AreEqual(numMessages * numThreads, result.Count);
Assert.AreEqual(1, result.First());
Assert.AreEqual(numMessages * numThreads, result.Last());
env.UndeployAll();
}
private static void TrySend(
RegressionEnvironment env,
int numThreads,
int numEventsPerThread)
{
var path = new RegressionPath();
var schemas = "create schema MyUpdateEvent as (key string, intupd int);\n" +
"create schema MySchema as (TheString string, intval int);\n";
env.CompileDeployWBusPublicType(schemas, path);
env.CompileDeploy("@Name('window') create window MyWindow#keepall as MySchema", path);
env.CompileDeploy(
"on MyUpdateEvent mue merge MyWindow mw " +
"where mw.TheString = mue.key " +
"when not matched then insert select key as TheString, intupd as intval " +
"when matched then delete",
path);
env.CompileDeploy(
"@Name('target') select (select intval from MyWindow mw where mw.TheString = sb.TheString) as val from SupportBean sb",
path);
// execute
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowSubqueryLookup)).ThreadFactory);
var future = new IFuture<bool?>[numThreads];
for (var i = 0; i < numThreads; i++) {
future[i] = threadPool.Submit(
new StmtNamedWindowSubqueryLookupCallable(
i,
env.Runtime,
numEventsPerThread,
env.Statement("target")));
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
var events = EPAssertionUtil.EnumeratorToArray(env.GetEnumerator("window"));
Assert.AreEqual(0, events.Length);
env.UndeployAll();
}
private static void TryChainedCountSum(
RegressionEnvironment env,
int numThreads,
int numEvents)
{
// setup statements
var path = new RegressionPath();
env.CompileDeploy("insert into MyStreamOne select count(*) as cnt from SupportBean", path);
env.CompileDeploy("@Name('s0') insert into MyStreamTwo select sum(cnt) as mysum from MyStreamOne", path);
var listener = new SupportUpdateListener();
env.Statement("s0").AddListener(listener);
// execute
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadDeterminismInsertInto)).ThreadFactory);
var future = new IFuture<object>[numThreads];
IReaderWriterLock sharedStartLock = new SlimReaderWriterLock();
using (sharedStartLock.WriteLock.Acquire()) {
for (var i = 0; i < numThreads; i++) {
future[i] = threadPool.Submit(
new SendEventRWLockCallable(
i,
sharedStartLock,
env.Runtime,
new GeneratorEnumerator(numEvents)));
}
SupportCompileDeployUtil.ThreadSleep(100);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// assert result
var newEvents = listener.NewDataListFlattened;
for (var i = 0; i < numEvents - 1; i++) {
var expected = Total(i + 1);
Assert.AreEqual(expected, newEvents[i].Get("mysum"));
}
env.UndeployAll();
}
private static void TryIterate(
RegressionEnvironment env,
RegressionPath path,
int numThreads,
int numRepeats)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowIterate)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtNamedWindowIterateCallable(Convert.ToString(i), env, path, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
}
private static void TrySendAndReceive(
RegressionEnvironment env,
int numThreads,
EPStatement statement,
int numRepeats)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtJoin)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtJoinCallable(i, env.Runtime, statement, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
}
private static void TrySend(
RegressionEnvironment env,
int numThreads,
int numRepeats)
{
env.CompileDeploy(
"@Name('s0') select (select Id from SupportBean_S0#length(1000000) where Id = S1.Id) as value from SupportBean_S1 as S1");
var listener = new SupportMTUpdateListener();
env.Statement("s0").AddListener(listener);
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtSubquery)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtSubqueryCallable(i, env.Runtime, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// Assert results
var totalExpected = numThreads * numRepeats;
// assert new data
var resultNewData = listener.GetNewDataListFlattened();
Assert.AreEqual(totalExpected, resultNewData.Length);
ISet<int> values = new HashSet<int>();
foreach (var theEvent in resultNewData) {
values.Add(theEvent.Get("value").AsInt32());
}
Assert.AreEqual(totalExpected, values.Count, "Unexpected duplicates");
listener.Reset();
env.UndeployAll();
}
private static void TrySend(
RegressionEnvironment env,
int numThreads,
int numEventsPerThread)
{
// setup statements
var path = new RegressionPath();
env.CompileDeploy("create window MyWindow#keepall as select * from SupportBean", path);
env.CompileDeploy(
"on SupportBean sb " +
"merge MyWindow nw where nw.TheString = sb.TheString " +
" when not matched then insert select * " +
" when matched then update set IntPrimitive = nw.IntPrimitive + 1",
path);
// execute
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowMerge)).ThreadFactory);
var future = new IFuture<bool?>[numThreads];
for (var i = 0; i < numThreads; i++) {
future[i] = threadPool.Submit(new StmtNamedWindowMergeCallable(env.Runtime, numEventsPerThread));
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// compare
var rows = env.CompileExecuteFAF("select * from MyWindow", path).Array;
Assert.AreEqual(numEventsPerThread, rows.Length);
foreach (var row in rows) {
Assert.AreEqual(numThreads - 1, row.Get("IntPrimitive"));
}
//long deltaTime = endTime - startTime;
//System.out.println("Totals updated: " + totalUpdates + " Delta cumu: " + deltaCumulative + " Delta pooled: " + deltaTime);
env.UndeployAll();
}
private static void TryListener(
RegressionEnvironment env,
int numThreads,
int numRepeats,
EPStatement stmt)
{
var executor = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtListenerCreateStmt)).ThreadFactory);
var futures = new List<IFuture<object>>();
for (var i = 0; i < numThreads; i++) {
var callable = new StmtListenerRouteCallable(i, env, stmt, numRepeats);
futures.Add(executor.Submit(callable));
}
// Give the futures 10 seconds to complete the futures...
futures.AsParallel().ForAll(future => future.Wait(TimeSpan.FromSeconds(10)));
executor.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(executor, TimeSpan.FromSeconds(10));
SupportCompileDeployUtil.AssertFutures(futures);
}
public void Run(RegressionEnvironment env)
{
var epl = "create window A#unique(Key) as MyEventA;\n" +
"create window B#unique(Key) as MyEventB;\n" +
"insert into A select * from MyEventA;\n" +
"insert into B select * from MyEventB;\n" +
"\n" +
"@Name('stmt') select sum(A.Data) as aTotal,sum(B.Data) as bTotal " +
"from A unidirectional, B where A.Key = B.Key;\n";
env.CompileDeploy(epl);
var es = Executors.NewFixedThreadPool(
10,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowJoinUniqueView)).ThreadFactory);
IList<MyRunnable> runnables = new List<MyRunnable>();
for (var i = 0; i < 6; i++) {
runnables.Add(new MyRunnable(env.EventService));
}
foreach (var toRun in runnables) {
es.Submit(toRun.Run);
}
SupportCompileDeployUtil.ThreadSleep(2000);
foreach (var toRun in runnables) {
toRun.Shutdown = true;
}
es.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(es, 20, TimeUnit.SECONDS);
foreach (var runnable in runnables) {
Assert.IsNull(runnable.Exception);
}
env.UndeployAll();
}
private static void TrySetAndReadAtomic(
RegressionEnvironment env,
SupportMTUpdateListener listenerSetOne,
SupportMTUpdateListener listenerSetTwo,
int numThreads,
int numRepeats)
{
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadVariables)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new VariableReadWriteCallable(i, env, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// Determine if we have all numbers for var3 and didn't skip one.
// Since "var3 = var3 + 1" is executed by multiple statements and threads we need to have
// this counter have all the values from 0 to N-1.
ISet<long> var3Values = new SortedSet<long>();
foreach (var theEvent in listenerSetOne.GetNewDataListFlattened()) {
var3Values.Add(theEvent.Get("var3").AsInt64());
}
foreach (var theEvent in listenerSetTwo.GetNewDataListFlattened()) {
var3Values.Add(theEvent.Get("var3").AsInt64());
}
Assert.AreEqual(numThreads * numRepeats, var3Values.Count);
for (var i = 1; i < numThreads * numRepeats + 1; i++) {
Assert.IsTrue(var3Values.Contains(i));
}
}
private static void TrySend(
RegressionEnvironment env,
int numThreads,
int numEventsPerThread)
{
// setup statements
var path = new RegressionPath();
env.CompileDeploy(
"create window MyWindow#unique(TheString, IntPrimitive) as select * from SupportBean",
path);
env.CompileDeploy("insert into MyWindow select * from SupportBean(BoolPrimitive = true)", path);
env.CompileDeploy(
"on SupportBean(BoolPrimitive = false) sb " +
"update MyWindow win set IntBoxed = win.IntBoxed + 1, DoublePrimitive = win.DoublePrimitive + sb.DoublePrimitive" +
" where sb.TheString = win.TheString and sb.IntPrimitive = win.IntPrimitive",
path);
// send primer events, initialize totals
IDictionary<Pair<string, int>, UpdateTotals> totals = new Dictionary<Pair<string, int>, UpdateTotals>();
for (var i = 0; i < NUM_STRINGS; i++) {
for (var j = 0; j < NUM_INTS; j++) {
var primer = new SupportBean(Convert.ToString(i), j);
primer.BoolPrimitive = true;
primer.IntBoxed = 0;
primer.DoublePrimitive = 0;
env.SendEventBean(primer);
var key = new Pair<string, int>(primer.TheString, primer.IntPrimitive);
totals.Put(key, new UpdateTotals(0, 0));
}
}
// execute
var startTime = PerformanceObserver.MilliTime;
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowUpdate)).ThreadFactory);
var future = new IFuture<StmtNamedWindowUpdateCallable.UpdateResult>[numThreads];
for (var i = 0; i < numThreads; i++) {
future[i] = threadPool.Submit(
new StmtNamedWindowUpdateCallable("Thread" + i, env.Runtime, numEventsPerThread));
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
var endTime = PerformanceObserver.MilliTime;
// total up result
long deltaCumulative = 0;
for (var i = 0; i < numThreads; i++) {
StmtNamedWindowUpdateCallable.UpdateResult result = null;
try {
result = future[i].GetValueOrDefault();
}
catch (Exception t) {
throw new EPException(t);
}
deltaCumulative += result.Delta;
foreach (var item in result.Updates) {
var key = new Pair<string, int>(item.TheString, item.Intval);
var total = totals.Get(key);
if (total == null) {
throw new EPException("Totals not found for key " + key);
}
total.Num = total.Num + 1;
total.Sum = total.Sum + item.DoublePrimitive;
}
}
// compare
var rows = env.CompileExecuteFAF("select * from MyWindow", path).Array;
Assert.AreEqual(rows.Length, totals.Count);
long totalUpdates = 0;
foreach (var row in rows) {
var key = new Pair<string, int>((string) row.Get("TheString"), row.Get("IntPrimitive").AsInt32());
var total = totals.Get(key);
Assert.AreEqual(total.Num, row.Get("IntBoxed"));
Assert.AreEqual(total.Sum, row.Get("DoublePrimitive"));
totalUpdates += total.Num;
}
Assert.AreEqual(totalUpdates, numThreads * numEventsPerThread);
//long deltaTime = endTime - startTime;
//System.out.println("Totals updated: " + totalUpdates + " Delta cumu: " + deltaCumulative + " Delta pooled: " + deltaTime);
env.UndeployAll();
}
private static void TrySend(
RegressionEnvironment env,
RegressionPath path,
SupportMTUpdateListener listenerWindow,
int numThreads,
int numRepeats,
int numConsumers)
{
var listenerConsumers = new SupportMTUpdateListener[numConsumers];
var compiled = env.Compile("select TheString, LongPrimitive from MyWindow", path);
for (var i = 0; i < listenerConsumers.Length; i++) {
var stmtName = "c" + i;
try {
env.Deployment.Deploy(compiled, new DeploymentOptions().WithStatementNameRuntime(ctx => stmtName));
}
catch (EPDeployException e) {
throw new EPException(e);
}
var stmtConsumer = env.Statement(stmtName);
listenerConsumers[i] = new SupportMTUpdateListener();
stmtConsumer.AddListener(listenerConsumers[i]);
}
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadStmtNamedWindowConsume)).ThreadFactory);
var future = new IFuture<object>[numThreads];
for (var i = 0; i < numThreads; i++) {
var callable = new StmtNamedWindowConsumeCallable(Convert.ToString(i), env.Runtime, numRepeats);
future[i] = threadPool.Submit(callable);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
// compute list of expected
IList<string> expectedIdsList = new List<string>();
for (var i = 0; i < numThreads; i++) {
try {
expectedIdsList.AddAll((IList<string>) future[i].Get());
}
catch (Exception t) {
throw new EPException(t);
}
}
var expectedIds = expectedIdsList.ToArray();
Assert.AreEqual(numThreads * numRepeats, listenerWindow.NewDataList.Count); // old and new each
// compute list of received
for (var i = 0; i < listenerConsumers.Length; i++) {
var newEvents = listenerConsumers[i].GetNewDataListFlattened();
var receivedIds = new string[newEvents.Length];
for (var j = 0; j < newEvents.Length; j++) {
receivedIds[j] = (string) newEvents[j].Get("TheString");
}
Assert.AreEqual(receivedIds.Length, expectedIds.Length);
Array.Sort(receivedIds);
Array.Sort(expectedIds);
CompatExtensions.DeepEqualsWithType(expectedIds, receivedIds);
}
}
private static void TryMultiInsertGroup(
RegressionEnvironment env,
int numThreads,
int numStatements,
int numEvents)
{
// This should fail all test in this class
// config.getEngineDefaults().getThreading().setInsertIntoDispatchPreserveOrder(false);
// setup statements
var path = new RegressionPath();
var insertIntoStmts = new EPStatement[numStatements];
for (var i = 0; i < numStatements; i++) {
var epl = $"@Name('s{i}') insert into MyStream select {i} as ident,count(*) as cnt from SupportBean";
insertIntoStmts[i] = env.CompileDeploy(epl, path).Statement("s" + i);
}
env.CompileDeploy("@Name('final') select ident, sum(cnt) as mysum from MyStream group by ident", path);
var listener = new SupportMTUpdateListener();
env.Statement("final").AddListener(listener);
// execute
var threadPool = Executors.NewFixedThreadPool(
numThreads,
new SupportThreadFactory(typeof(MultithreadDeterminismInsertInto)).ThreadFactory);
var future = new IFuture<object>[numThreads];
var sharedStartLock = new SlimReaderWriterLock();
using (sharedStartLock.WriteLock.Acquire()) {
for (var i = 0; i < numThreads; i++) {
future[i] = threadPool.Submit(
new SendEventRWLockCallable(
i,
sharedStartLock,
env.Runtime,
new GeneratorEnumerator(numEvents)));
}
SupportCompileDeployUtil.ThreadSleep(100);
}
threadPool.Shutdown();
SupportCompileDeployUtil.ExecutorAwait(threadPool, 10, TimeUnit.SECONDS);
SupportCompileDeployUtil.AssertFutures(future);
// assert result
var newEvents = listener.GetNewDataListFlattened();
IList<long>[] resultsPerIdent = new List<long>[numStatements];
foreach (var theEvent in newEvents) {
var ident = theEvent.Get("ident").AsInt32();
if (resultsPerIdent[ident] == null) {
resultsPerIdent[ident] = new List<long>();
}
var mysum = theEvent.Get("mysum").AsInt64();
resultsPerIdent[ident].Add(mysum);
}
for (var statement = 0; statement < numStatements; statement++) {
for (var i = 0; i < numEvents - 1; i++) {
var expected = Total(i + 1);
Assert.AreEqual(
expected,
resultsPerIdent[statement][i],
"Failed for statement " + statement);
}
}
env.UndeployAll();
}