private void RunAssertionNonBlockingJoinException(EPServiceProvider epService)
{
epService.EPAdministrator.CreateEPL("create schema SomeType ()");
epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowOne " +
"DefaultSupportSourceOp -> outstream<SomeType> {}" +
"DefaultSupportCaptureOp(outstream) {}");
var latchOne = new CountDownLatch(1);
var src = new DefaultSupportSourceOp(new object[] {latchOne, new MyRuntimeException("TestException")});
var output = new DefaultSupportCaptureOp(SupportContainer.Instance.LockManager());
var options =
new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(src, output));
var dfOne = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options);
dfOne.Start();
var unlatchingThread = new Thread(
() =>
{
try
{
Thread.Sleep(300);
latchOne.CountDown();
}
catch (Exception e)
{
Console.Error.WriteLine(e.StackTrace);
}
});
unlatchingThread.Start();
dfOne.Join();
Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State);
Assert.AreEqual(0, output.GetAndReset().Count);
epService.EPAdministrator.DestroyAllStatements();
}
private void RunAssertionNonBlockingJoinMultipleRunnable(EPServiceProvider epService)
{
// declare
epService.EPAdministrator.CreateEPL("create schema SomeType ()");
epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowOne " +
"SourceOne -> outstream<SomeType> {}" +
"SourceTwo -> outstream<SomeType> {}" +
"Future(outstream) {}");
// instantiate
var latchOne = new CountDownLatch(1);
var latchTwo = new CountDownLatch(1);
var ops = new Dictionary<string, object>();
ops.Put("SourceOne", new DefaultSupportSourceOp(new object[] {latchOne, new object[] {1}}));
ops.Put("SourceTwo", new DefaultSupportSourceOp(new object[] {latchTwo, new object[] {1}}));
var future = new DefaultSupportCaptureOp(2, SupportContainer.Instance.LockManager());
ops.Put("Future", future);
var options =
new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProviderByOpName(ops));
var dfOne = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options);
dfOne.Start();
Thread.Sleep(50);
Assert.AreEqual(EPDataFlowState.RUNNING, dfOne.State);
latchOne.CountDown();
Thread.Sleep(200);
Assert.AreEqual(EPDataFlowState.RUNNING, dfOne.State);
latchTwo.CountDown();
dfOne.Join();
Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State);
Assert.AreEqual(2, future.GetAndReset().Count);
epService.EPAdministrator.DestroyAllStatements();
}
private void TrySource(Supplier <AdapterInputSource> sourceSupplier)
{
var source = sourceSupplier.Invoke();
var spec = new CSVInputAdapterSpec(source, "TypeC");
_runtime = EPRuntimeProvider.GetRuntime("testPlayFromInputStream", MakeConfig("TypeC"));
_runtime.Initialize();
InputAdapter feed = new CSVInputAdapter(_runtime, spec);
var stmt = CompileDeploy(_runtime, "select * from TypeC#length(100)").Statements[0];
var listener = new SupportUpdateListener();
stmt.Events += listener.Update;
feed.Start();
Assert.AreEqual(1, listener.GetNewDataList().Count);
source = sourceSupplier.Invoke();
// test graph
var graph = "create dataflow ReadCSV " +
"FileSource -> mystream<TypeC> { hasTitleLine: true }" +
"DefaultSupportCaptureOp(mystream) {}";
var deployment = CompileDeploy(_runtime, graph);
var outputOp = new DefaultSupportCaptureOp();
var options = new EPDataFlowInstantiationOptions();
options.OperatorProvider = new DefaultSupportGraphOpProvider(outputOp);
options.ParameterProvider = new DefaultSupportGraphParamProvider(Collections.SingletonDataMap("adapterInputSource", source));
var instance = _runtime.DataFlowService.Instantiate(deployment.DeploymentId, "ReadCSV", options);
instance.Run();
var received = outputOp.GetAndReset()[0].ToArray();
Assert.AreEqual(1, received.Length);
}
private void TryAssertionJoinOrder(EPServiceProvider epService, string fromClause)
{
string graph = "create dataflow MySelect\n" +
"Emitter -> instream_s0<SupportBean_S0>{name: 'emitterS0'}\n" +
"Emitter -> instream_s1<SupportBean_S1>{name: 'emitterS1'}\n" +
"Emitter -> instream_s2<SupportBean_S2>{name: 'emitterS2'}\n" +
"Select(instream_s0 as S0, instream_s1 as S1, instream_s2 as S2) -> outstream {\n" +
" select: (select s0.id as s0id, s1.id as s1id, s2.id as s2id " + fromClause + ")\n" +
"}\n" +
"CaptureOp(outstream) {}\n";
EPStatement stmtGraph = epService.EPAdministrator.CreateEPL(graph);
var capture = new DefaultSupportCaptureOp(SupportContainer.Instance.LockManager());
IDictionary <string, Object> operators = CollectionUtil.PopulateNameValueMap("CaptureOp", capture);
var options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProviderByOpName(operators));
EPDataFlowInstance instance = epService.EPRuntime.DataFlowRuntime.Instantiate("MySelect", options);
EPDataFlowInstanceCaptive captive = instance.StartCaptive();
captive.Emitters.Get("emitterS0").Submit(new SupportBean_S0(1));
captive.Emitters.Get("emitterS1").Submit(new SupportBean_S1(10));
Assert.AreEqual(0, capture.Current.Length);
captive.Emitters.Get("emitterS2").Submit(new SupportBean_S2(100));
Assert.AreEqual(1, capture.Current.Length);
EPAssertionUtil.AssertProps(
epService.Container, capture.GetCurrentAndReset()[0], "s0id,s1id,s2id".Split(','), new object[] { 1, 10, 100 });
instance.Cancel();
captive.Emitters.Get("emitterS2").Submit(new SupportBean_S2(101));
Assert.AreEqual(0, capture.Current.Length);
stmtGraph.Dispose();
}
private void RunAssertionAllTypes(String typeName, Object[] events)
{
String graph = "create dataflow MySelect\n" +
"DefaultSupportSourceOp -> instream<" + typeName + ">{}\n" +
"Select(instream as ME) -> outstream {select: (select myString, sum(myInt) as total from ME)}\n" +
"DefaultSupportCaptureOp(outstream) {}";
EPStatement stmtGraph = _epService.EPAdministrator.CreateEPL(graph);
DefaultSupportSourceOp source = new DefaultSupportSourceOp(events);
DefaultSupportCaptureOp capture = new DefaultSupportCaptureOp(2);
EPDataFlowInstantiationOptions options = new EPDataFlowInstantiationOptions();
options.OperatorProvider(new DefaultSupportGraphOpProvider(source, capture));
EPDataFlowInstance instance = _epService.EPRuntime.DataFlowRuntime.Instantiate("MySelect", options);
instance.Run();
Object[] result = capture.GetAndReset()[0].ToArray();
EPAssertionUtil.AssertPropsPerRow(result, "myString,total".Split(','), new Object[][] { new Object[] { "one", 1 }, new Object[] { "two", 3 } });
instance.Cancel();
stmtGraph.Dispose();
}
private void RunAssertionSendEventDynamicType(EPServiceProvider epService)
{
epService.EPAdministrator.CreateEPL("create objectarray schema MyEventOne(type string, p0 int, p1 string)");
epService.EPAdministrator.CreateEPL("create objectarray schema MyEventTwo(type string, f0 string, f1 int)");
var listener = new SupportUpdateListener();
epService.EPAdministrator.CreateEPL("select * from MyEventOne").Events += listener.Update;
var listenerTwo = new SupportUpdateListener();
epService.EPAdministrator.CreateEPL("select * from MyEventTwo").Events += listenerTwo.Update;
epService.EPAdministrator.CreateEPL("create dataflow MyDataFlow " +
"MyObjectArrayGraphSource -> OutStream<?> {}" +
"EventBusSink(OutStream) {" +
" collector : {" +
" class: '" + typeof(MyTransformToEventBus).FullName + "'" +
" }" +
"}");
var source = new MyObjectArrayGraphSource(Collections.List(
new object[] { "type1", 100, "abc" },
new object[] { "type2", "GE", -1 }
).GetEnumerator());
var options = new EPDataFlowInstantiationOptions()
.OperatorProvider(new DefaultSupportGraphOpProvider(source));
epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlow", options).Start();
listener.WaitForInvocation(3000, 1);
listenerTwo.WaitForInvocation(3000, 1);
EPAssertionUtil.AssertProps(listener.AssertOneGetNewAndReset(), "p0,p1".Split(','), new object[] { 100, "abc" });
EPAssertionUtil.AssertProps(listenerTwo.AssertOneGetNewAndReset(), "f0,f1".Split(','), new object[] { "GE", -1 });
epService.EPAdministrator.DestroyAllStatements();
}
public void SaveConfiguration(
string dataflowConfigName,
string deploymentId,
string dataFlowName,
EPDataFlowInstantiationOptions options)
{
lock (this) {
var entry = GetEntryMayNull(deploymentId, dataFlowName);
if (entry == null) {
var message = "Failed to locate data flow '" + dataFlowName + "'";
throw new EPDataFlowNotFoundException(message);
}
if (_configurationState.Exists(dataflowConfigName)) {
var message = "Data flow saved configuration by name '" +
dataflowConfigName +
"' already exists";
throw new EPDataFlowAlreadyExistsException(message);
}
_configurationState.Add(
new EPDataFlowSavedConfiguration(dataflowConfigName, deploymentId, dataFlowName, options));
}
}
public void TestIterateFinalMarker()
{
_epService.EPAdministrator.Configuration.AddEventType <SupportBean>();
String graph = "create dataflow MySelect\n" +
"Emitter -> instream_s0<SupportBean>{name: 'emitterS0'}\n" +
"@Audit Select(instream_s0 as ALIAS) -> outstream {\n" +
" select: (select TheString, sum(IntPrimitive) as SumInt from ALIAS group by TheString order by TheString asc),\n" +
" iterate: true" +
"}\n" +
"CaptureOp(outstream) {}\n";
_epService.EPRuntime.SendEvent(new CurrentTimeEvent(0));
_epService.EPAdministrator.CreateEPL(graph);
DefaultSupportCaptureOp capture = new DefaultSupportCaptureOp();
IDictionary <String, Object> operators = CollectionUtil.PopulateNameValueMap("CaptureOp", capture);
EPDataFlowInstantiationOptions options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProviderByOpName(operators));
EPDataFlowInstance instance = _epService.EPRuntime.DataFlowRuntime.Instantiate("MySelect", options);
EPDataFlowInstanceCaptive captive = instance.StartCaptive();
Emitter emitter = captive.Emitters.Get("emitterS0");
emitter.Submit(new SupportBean("E3", 4));
emitter.Submit(new SupportBean("E2", 3));
emitter.Submit(new SupportBean("E1", 1));
emitter.Submit(new SupportBean("E2", 2));
emitter.Submit(new SupportBean("E1", 5));
Assert.AreEqual(0, capture.GetCurrent().Length);
emitter.SubmitSignal(new EPDataFlowSignalFinalMarkerImpl());
EPAssertionUtil.AssertPropsPerRow(capture.GetCurrent(), "TheString,SumInt".Split(','), new Object[][] { new Object[] { "E1", 6 }, new Object[] { "E2", 5 }, new Object[] { "E3", 4 } });
instance.Cancel();
}
public void Run(RegressionEnvironment env)
{
// declare
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowOne " +
"BeaconSource -> BeaconStream {iterations : 1}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
// instantiate
var future = new DefaultSupportCaptureOp(1, env.Container.LockManager());
var options =
new EPDataFlowInstantiationOptions().WithOperatorProvider(
new DefaultSupportGraphOpProvider(future));
var dfOne = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options);
Assert.AreEqual("MyDataFlowOne", dfOne.DataFlowName);
Assert.AreEqual(EPDataFlowState.INSTANTIATED, dfOne.State);
// has not run
Sleep(1000);
Assert.IsFalse(future.IsDone());
// blocking run
dfOne.Run();
Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State);
try {
Assert.AreEqual(1, future.GetValueOrDefault().Length);
}
catch (Exception t) {
throw new EPException(t);
}
// assert past-exec
TryAssertionAfterExec(dfOne);
env.UndeployAll();
}
private void RunAssertionBeaconNoType(EPServiceProvider epService)
{
EPDataFlowInstantiationOptions options;
object[] output;
epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowOne " +
"BeaconSource -> BeaconStream {}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var countExpected = 10;
var futureAtLeast = new DefaultSupportCaptureOp(countExpected, SupportContainer.Instance.LockManager());
options = new EPDataFlowInstantiationOptions()
.OperatorProvider(new DefaultSupportGraphOpProvider(futureAtLeast));
var df = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options);
df.Start();
output = futureAtLeast.GetValue(1, TimeUnit.SECONDS);
Assert.IsTrue(countExpected <= output.Length);
df.Cancel();
// BeaconSource with given number of iterations
epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowTwo " +
"BeaconSource -> BeaconStream {" +
" iterations: 5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureExactTwo = new DefaultSupportCaptureOp(5, SupportContainer.Instance.LockManager());
options = new EPDataFlowInstantiationOptions()
.OperatorProvider(new DefaultSupportGraphOpProvider(futureExactTwo));
epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowTwo", options).Start();
output = futureExactTwo.GetValue(1, TimeUnit.SECONDS);
Assert.AreEqual(5, output.Length);
// BeaconSource with delay
epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowThree " +
"BeaconSource -> BeaconStream {" +
" iterations: 2," +
" initialDelay: 0.5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureExactThree = new DefaultSupportCaptureOp(2, SupportContainer.Instance.LockManager());
options = new EPDataFlowInstantiationOptions()
.OperatorProvider(new DefaultSupportGraphOpProvider(futureExactThree));
var start = PerformanceObserver.MilliTime;
epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowThree", options).Start();
output = futureExactThree.GetValue(1, TimeUnit.SECONDS);
var end = PerformanceObserver.MilliTime;
Assert.AreEqual(2, output.Length);
Assert.IsTrue(end - start > 490, "delta=" + (end - start));
// BeaconSource with period
epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowFour " +
"BeaconSource -> BeaconStream {" +
" interval: 0.5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureFour = new DefaultSupportCaptureOp(2, SupportContainer.Instance.LockManager());
options = new EPDataFlowInstantiationOptions()
.OperatorProvider(new DefaultSupportGraphOpProvider(futureFour));
epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFour", options).Start();
output = futureFour.GetValue(1, TimeUnit.SECONDS);
Assert.AreEqual(2, output.Length);
// test Beacon with define typed
epService.EPAdministrator.CreateEPL("create objectarray schema MyTestOAType(p1 string)");
epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowFive " +
"BeaconSource -> BeaconStream<MyTestOAType> {" +
" interval: 0.5," +
" p1 : 'abc'" +
"}");
epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFive");
}
private void RunAssertionFields(
EPServiceProvider epService, EventRepresentationChoice representationEnum, bool eventbean)
{
EPDataFlowInstantiationOptions options;
epService.EPAdministrator.CreateEPL(
"create " + representationEnum.GetOutputTypeCreateSchemaName() +
" schema MyEvent(p0 string, p1 long, p2 double)");
var stmtGraph = epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowOne " +
"" +
"BeaconSource -> BeaconStream<" + (eventbean ? "EventBean<MyEvent>" : "MyEvent") + "> {" +
" iterations : 3," +
" p0 : 'abc'," +
" p1 : MyMath.Round(MyMath.Random() * 10) + 1," +
" p2 : 1d," +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var future = new DefaultSupportCaptureOp(3, SupportContainer.Instance.LockManager());
options = new EPDataFlowInstantiationOptions()
.OperatorProvider(new DefaultSupportGraphOpProvider(future));
var df = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options);
df.Start();
var output = future.GetValue(2, TimeUnit.SECONDS);
Assert.AreEqual(3, output.Length);
for (var i = 0; i < 3; i++)
{
if (!eventbean)
{
if (representationEnum.IsObjectArrayEvent())
{
var row = (object[])output[i];
Assert.AreEqual("abc", row[0]);
var val = (long)row[1];
Assert.IsTrue(val >= 0 && val <= 11, "val=" + val);
Assert.AreEqual(1d, row[2]);
}
else if (representationEnum.IsMapEvent())
{
Map row = (Map)output[i];
Assert.AreEqual("abc", row.Get("p0"));
var val = (long)row.Get("p1");
Assert.IsTrue(val >= 0 && val <= 11, "val=" + val);
Assert.AreEqual(1d, row.Get("p2"));
}
else
{
var row = (GenericRecord)output[i];
Assert.AreEqual("abc", row.Get("p0"));
var val = (long)row.Get("p1");
Assert.IsTrue(val >= 0 && val <= 11, "val=" + val);
Assert.AreEqual(1d, row.Get("p2"));
}
}
else
{
var row = (EventBean)output[i];
Assert.AreEqual("abc", row.Get("p0"));
}
}
stmtGraph.Dispose();
epService.EPAdministrator.Configuration.RemoveEventType("MyEvent", true);
}
public void Run(RegressionEnvironment env)
{
var fields = new [] { "p0","p1" };
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlow " +
"Emitter -> outstream<MyOAEventType> {name:'src1'}" +
"DefaultSupportCaptureOp(outstream) {}");
var captureOp = new DefaultSupportCaptureOp();
var options = new EPDataFlowInstantiationOptions();
options.WithOperatorProvider(new DefaultSupportGraphOpProvider(captureOp));
var instance = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlow", options);
var captiveStart = instance.StartCaptive();
Assert.AreEqual(0, captiveStart.Runnables.Count);
Assert.AreEqual(1, captiveStart.Emitters.Count);
var emitter = captiveStart.Emitters.Get("src1");
Assert.AreEqual(EPDataFlowState.RUNNING, instance.State);
emitter.Submit(new object[] {"E1", 10});
EPAssertionUtil.AssertPropsPerRow(
env.Container,
captureOp.Current,
fields,
new[] {new object[] {"E1", 10}});
emitter.Submit(new object[] {"E2", 20});
EPAssertionUtil.AssertPropsPerRow(
env.Container,
captureOp.Current,
fields,
new[] {new object[] {"E1", 10}, new object[] {"E2", 20}});
emitter.SubmitSignal(new EPDataFlowSignalFinalMarkerImpl());
EPAssertionUtil.AssertPropsPerRow(
env.Container,
captureOp.Current,
fields,
new object[0][]);
EPAssertionUtil.AssertPropsPerRow(
env.Container,
captureOp.GetAndReset()[0].UnwrapIntoArray<object>(),
fields,
new[] {new object[] {"E1", 10}, new object[] {"E2", 20}});
emitter.Submit(new object[] {"E3", 30});
EPAssertionUtil.AssertPropsPerRow(
env.Container,
captureOp.Current,
fields,
new[] {new object[] {"E3", 30}});
// stays running until cancelled (no transition to complete)
Assert.AreEqual(EPDataFlowState.RUNNING, instance.State);
instance.Cancel();
Assert.AreEqual(EPDataFlowState.CANCELLED, instance.State);
env.UndeployAll();
// test doc sample
var epl = "@Name('flow') create dataflow HelloWorldDataFlow\n" +
" create schema SampleSchema(text string),\t// sample type\t\t\n" +
"\t\n" +
" Emitter -> helloworld.stream<SampleSchema> { name: 'myemitter' }\n" +
" LogSink(helloworld.stream) {}";
env.CompileDeploy(epl);
env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "HelloWorldDataFlow");
env.UndeployAll();
}
public void Run(RegressionEnvironment env)
{
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowOne " +
"create map schema SingleProp (Id string), " +
"EPStatementSource -> thedata<SingleProp> {" +
" statementDeploymentId : 'MyDeploymentId'," +
" statementName : 'MyStatement'," +
"} " +
"DefaultSupportCaptureOp(thedata) {}");
var captureOp = new DefaultSupportCaptureOp();
var options = new EPDataFlowInstantiationOptions()
.WithOperatorProvider(new DefaultSupportGraphOpProvider(captureOp));
var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options);
Assert.IsNull(df.UserObject);
Assert.IsNull(df.InstanceId);
df.Start();
env.SendEventBean(new SupportBean("E1", 1));
Assert.AreEqual(0, captureOp.Current.Length);
var epl = "@Name('MyStatement') select TheString as Id from SupportBean";
var compiled = env.Compile(epl);
try {
env.Deployment.Deploy(compiled, new DeploymentOptions().WithDeploymentId("MyDeploymentId"));
}
catch (EPDeployException e) {
Assert.Fail(e.Message);
}
env.SendEventBean(new SupportBean("E2", 2));
captureOp.WaitForInvocation(100, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "Id" },
new object[] {"E2"});
env.UndeployModuleContaining("MyStatement");
env.SendEventBean(new SupportBean("E3", 3));
Assert.AreEqual(0, captureOp.Current.Length);
try {
env.Deployment.Deploy(compiled, new DeploymentOptions().WithDeploymentId("MyDeploymentId"));
}
catch (EPDeployException e) {
Assert.Fail(e.Message);
}
env.SendEventBean(new SupportBean("E4", 4));
captureOp.WaitForInvocation(100, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "Id" },
new object[] {"E4"});
env.UndeployModuleContaining("MyStatement");
env.SendEventBean(new SupportBean("E5", 5));
Assert.AreEqual(0, captureOp.Current.Length);
compiled = env.Compile("@Name('MyStatement') select 'X'||TheString||'X' as Id from SupportBean");
try {
env.Deployment.Deploy(compiled, new DeploymentOptions().WithDeploymentId("MyDeploymentId"));
}
catch (EPDeployException e) {
Assert.Fail(e.Message);
}
env.SendEventBean(new SupportBean("E6", 6));
captureOp.WaitForInvocation(100, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "Id" },
new object[] {"XE6X"});
df.Cancel();
env.UndeployAll();
}
public void TestBeaconNoType()
{
EPDataFlowInstantiationOptions options;
Object[] output;
_epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowOne " +
"BeaconSource -> BeaconStream {}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var countExpected = 10;
var futureAtLeast = new DefaultSupportCaptureOp(countExpected);
options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureAtLeast));
var df = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options);
df.Start();
output = futureAtLeast.GetValue(TimeSpan.FromSeconds(1));
Assert.IsTrue(countExpected <= output.Length);
df.Cancel();
// BeaconSource with given number of iterations
_epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowTwo " +
"BeaconSource -> BeaconStream {" +
" iterations: 5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureExactTwo = new DefaultSupportCaptureOp(5);
options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureExactTwo));
_epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowTwo", options).Start();
output = futureExactTwo.GetValue(TimeSpan.FromSeconds(1));
Assert.AreEqual(5, output.Length);
// BeaconSource with delay
_epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowThree " +
"BeaconSource -> BeaconStream {" +
" iterations: 2," +
" initialDelay: 0.5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureExactThree = new DefaultSupportCaptureOp(2);
options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureExactThree));
long start = Environment.TickCount;
_epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowThree", options).Start();
output = futureExactThree.GetValue(TimeSpan.FromSeconds(1));
long end = Environment.TickCount;
Assert.AreEqual(2, output.Length);
Assert.IsTrue(end - start > 490, "delta=" + (end - start));
// BeaconSource with period
_epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowFour " +
"BeaconSource -> BeaconStream {" +
" interval: 0.5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureFour = new DefaultSupportCaptureOp(2);
options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureFour));
_epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFour", options).Start();
output = futureFour.GetValue(TimeSpan.FromSeconds(1));
Assert.AreEqual(2, output.Length);
// test Beacon with define typed
_epService.EPAdministrator.CreateEPL("create objectarray schema MyTestOAType(p1 string)");
_epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowFive " +
"BeaconSource -> BeaconStream<MyTestOAType> {" +
" interval: 0.5," +
" p1 : 'abc'" +
"}");
_epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFive");
}
private static EPDataFlowEmitter GenerateRuntimeContext(
AgentInstanceContext agentInstanceContext,
DataflowDesc dataflow,
string instanceId,
int producerOpNum,
string operatorPrettyPrint,
DataFlowSignalManager dataFlowSignalManager,
IList<ObjectBindingPair>[] targetsPerStream,
EPDataFlowInstantiationOptions options)
{
// handle no targets
if (targetsPerStream == null) {
return new EPDataFlowEmitterNoTarget(producerOpNum, dataFlowSignalManager);
}
var dataflowName = dataflow.DataflowName;
var classpathImportService = agentInstanceContext.ImportServiceRuntime;
// handle single-stream case
if (targetsPerStream.Length == 1) {
var targets = targetsPerStream[0];
// handle single-stream single target case
if (targets.Count == 1) {
var target = targets[0];
return GetSubmitHandler(
agentInstanceContext,
dataflow.DataflowName,
instanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
target,
options.ExceptionHandler,
classpathImportService);
}
var handlers = new SubmitHandler[targets.Count];
for (var i = 0; i < handlers.Length; i++) {
handlers[i] = GetSubmitHandler(
agentInstanceContext,
dataflowName,
instanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
targets[i],
options.ExceptionHandler,
classpathImportService);
}
return new EPDataFlowEmitter1StreamNTarget(producerOpNum, dataFlowSignalManager, handlers);
}
// handle multi-stream case
var handlersPerStream = new SubmitHandler[targetsPerStream.Length][];
for (var streamNum = 0; streamNum < targetsPerStream.Length; streamNum++) {
var handlers = new SubmitHandler[targetsPerStream[streamNum].Count];
handlersPerStream[streamNum] = handlers;
for (var i = 0; i < handlers.Length; i++) {
handlers[i] = GetSubmitHandler(
agentInstanceContext,
dataflowName,
instanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
targetsPerStream[streamNum][i],
options.ExceptionHandler,
classpathImportService);
}
}
return new EPDataFlowEmitterNStreamNTarget(producerOpNum, dataFlowSignalManager, handlersPerStream);
}
public void Run(RegressionEnvironment env)
{
SupportGraphSource.GetAndResetLifecycle();
var compiled = env.Compile(
"@Name('flow') create dataflow MyDataFlow @Name('Goodie') @Audit SupportGraphSource -> outstream<SupportBean> {propOne:'abc'}");
var events = SupportGraphSourceForge.GetAndResetLifecycle();
Assert.AreEqual(3, events.Count);
Assert.AreEqual("instantiated", events[0]);
Assert.AreEqual("SetPropOne=abc", events[1]);
var forgeCtx = (DataFlowOpForgeInitializeContext) events[2];
Assert.AreEqual(0, forgeCtx.InputPorts.Count);
Assert.AreEqual(1, forgeCtx.OutputPorts.Count);
Assert.AreEqual("outstream", forgeCtx.OutputPorts[0].StreamName);
Assert.AreEqual("SupportBean", forgeCtx.OutputPorts[0].OptionalDeclaredType.EventType.Name);
Assert.AreEqual(2, forgeCtx.OperatorAnnotations.Length);
Assert.AreEqual("Goodie", ((NameAttribute) forgeCtx.OperatorAnnotations[0]).Value);
Assert.IsNotNull((AuditAttribute) forgeCtx.OperatorAnnotations[1]);
Assert.AreEqual("MyDataFlow", forgeCtx.DataflowName);
Assert.AreEqual(0, forgeCtx.OperatorNumber);
env.Deploy(compiled);
events = SupportGraphSourceFactory.GetAndResetLifecycle();
Assert.AreEqual(3, events.Count);
Assert.AreEqual("instantiated", events[0]);
Assert.AreEqual("SetPropOne=abc", events[1]);
var factoryCtx = (DataFlowOpFactoryInitializeContext) events[2];
Assert.AreEqual("MyDataFlow", factoryCtx.DataFlowName);
Assert.AreEqual(0, factoryCtx.OperatorNumber);
Assert.IsNotNull(factoryCtx.StatementContext);
// instantiate
var options = new EPDataFlowInstantiationOptions()
.WithDataFlowInstanceId("id1")
.WithDataFlowInstanceUserObject("myobject");
var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlow", options);
events = SupportGraphSourceFactory.GetAndResetLifecycle();
Assert.AreEqual(1, events.Count);
var opCtx = (DataFlowOpInitializeContext) events[0];
Assert.AreEqual("MyDataFlow", opCtx.DataFlowName);
Assert.AreEqual("id1", opCtx.DataFlowInstanceId);
Assert.IsNotNull(opCtx.AgentInstanceContext);
Assert.AreEqual("myobject", opCtx.DataflowInstanceUserObject);
Assert.AreEqual(0, opCtx.OperatorNumber);
Assert.AreEqual("SupportGraphSource", opCtx.OperatorName);
events = SupportGraphSource.GetAndResetLifecycle();
Assert.AreEqual(1, events.Count);
Assert.AreEqual("instantiated", events[0]); // instantiated
// run
df.Run();
events = SupportGraphSource.GetAndResetLifecycle();
Assert.AreEqual(5, events.Count);
Assert.IsTrue(events[0] is DataFlowOpOpenContext); // called open (GraphSource only)
Assert.AreEqual("next(numrows=0)", events[1]);
Assert.AreEqual("next(numrows=1)", events[2]);
Assert.AreEqual("next(numrows=2)", events[3]);
Assert.IsTrue(events[4] is DataFlowOpCloseContext); // called close (GraphSource only)
env.UndeployAll();
}
public void Run(RegressionEnvironment env)
{
EPDataFlowInstantiationOptions options;
object[] output;
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowOne " +
"BeaconSource -> BeaconStream {}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var countExpected = 10;
var futureAtLeast = new DefaultSupportCaptureOp(countExpected, env.Container.LockManager());
options = new EPDataFlowInstantiationOptions()
.WithOperatorProvider(new DefaultSupportGraphOpProvider(futureAtLeast));
var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options);
df.Start();
try {
output = futureAtLeast.GetValue(1, TimeUnit.SECONDS);
}
catch (Exception e) {
throw new EPException(e);
}
Assert.IsTrue(countExpected <= output.Length);
df.Cancel();
env.UndeployAll();
// BeaconSource with given number of iterations
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowTwo " +
"BeaconSource -> BeaconStream {" +
" iterations: 5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureExactTwo = new DefaultSupportCaptureOp(5, env.Container.LockManager());
options = new EPDataFlowInstantiationOptions()
.WithOperatorProvider(new DefaultSupportGraphOpProvider(futureExactTwo));
env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowTwo", options).Start();
try {
output = futureExactTwo.GetValue(1, TimeUnit.SECONDS);
}
catch (Exception t) {
throw new EPException(t);
}
Assert.AreEqual(5, output.Length);
env.UndeployAll();
// BeaconSource with delay
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowThree " +
"BeaconSource -> BeaconStream {" +
" iterations: 2," +
" initialDelay: 0.5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureExactThree = new DefaultSupportCaptureOp(2, env.Container.LockManager());
options = new EPDataFlowInstantiationOptions()
.WithOperatorProvider(new DefaultSupportGraphOpProvider(futureExactThree));
var start = PerformanceObserver.MilliTime;
env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowThree", options).Start();
try {
output = futureExactThree.GetValue(1, TimeUnit.SECONDS);
}
catch (Exception e) {
throw new EPException(e);
}
var end = PerformanceObserver.MilliTime;
Assert.AreEqual(2, output.Length);
Assert.IsTrue(end - start > 490, "delta=" + (end - start));
env.UndeployAll();
// BeaconSource with period
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowFour " +
"BeaconSource -> BeaconStream {" +
" interval: 0.5" +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}");
var futureFour = new DefaultSupportCaptureOp(2, env.Container.LockManager());
options = new EPDataFlowInstantiationOptions()
.WithOperatorProvider(new DefaultSupportGraphOpProvider(futureFour));
var instance = env.Runtime.DataFlowService.Instantiate(
env.DeploymentId("flow"),
"MyDataFlowFour",
options);
instance.Start();
try {
output = futureFour.GetValue(2, TimeUnit.SECONDS);
}
catch (Exception t) {
throw new EPException(t);
}
Assert.AreEqual(2, output.Length);
instance.Cancel();
env.UndeployAll();
// test Beacon with define typed
var path = new RegressionPath();
env.CompileDeploy("create objectarray schema MyTestOAType(p1 string)", path);
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowFive " +
"BeaconSource -> BeaconStream<MyTestOAType> {" +
" interval: 0.5," +
" p1 : 'abc'" +
"}",
path);
env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowFive");
env.UndeployAll();
}
public static EPDataFlowInstance Instantiate(
IContainer container,
int agentInstanceId,
DataflowDesc dataflow,
EPDataFlowInstantiationOptions options)
{
var statementContext = dataflow.StatementContext;
// allocate agent instance context
var @lock = statementContext.StatementAgentInstanceLockFactory.GetStatementLock(
statementContext.StatementName,
statementContext.Annotations,
statementContext.IsStatelessSelect,
statementContext.StatementType);
var handle = new EPStatementAgentInstanceHandle(statementContext.EpStatementHandle, agentInstanceId, @lock);
var auditProvider = statementContext.StatementInformationals.AuditProvider;
var instrumentationProvider = statementContext.StatementInformationals.InstrumentationProvider;
var agentInstanceContext = new AgentInstanceContext(statementContext, handle, null, null, auditProvider, instrumentationProvider);
// assure variables
statementContext.VariableManagementService.SetLocalVersion();
// instantiate operators
var operators = InstantiateOperators(container, agentInstanceContext, options, dataflow);
// determine binding of each channel to input methods (ports)
IList<LogicalChannelBinding> operatorChannelBindings = new List<LogicalChannelBinding>();
foreach (var channel in dataflow.LogicalChannels) {
var targetClass = operators.Get(channel.ConsumingOpNum).GetType();
var consumingMethod = FindMatchingMethod(channel.ConsumingOpPrettyPrint, targetClass, channel, false);
LogicalChannelBindingMethodDesc onSignalMethod = null;
if (channel.OutputPort.HasPunctuation) {
onSignalMethod = FindMatchingMethod(channel.ConsumingOpPrettyPrint, targetClass, channel, true);
}
operatorChannelBindings.Add(new LogicalChannelBinding(channel, consumingMethod, onSignalMethod));
}
// obtain realization
var dataFlowSignalManager = new DataFlowSignalManager();
var statistics = DataflowInstantiatorHelper.Realize(
dataflow,
operators,
operatorChannelBindings,
dataFlowSignalManager,
options,
agentInstanceContext);
// For each GraphSource add runnable
IList<GraphSourceRunnable> sourceRunnables = new List<GraphSourceRunnable>();
var audit = AuditEnum.DATAFLOW_SOURCE.GetAudit(statementContext.Annotations) != null;
foreach (var operatorEntry in operators) {
if (!(operatorEntry.Value is DataFlowSourceOperator)) {
continue;
}
var meta = dataflow.OperatorMetadata.Get(operatorEntry.Key);
var graphSource = (DataFlowSourceOperator) operatorEntry.Value;
var runnable = new GraphSourceRunnable(
agentInstanceContext,
graphSource,
dataflow.DataflowName,
options.DataFlowInstanceId,
meta.OperatorName,
operatorEntry.Key,
meta.OperatorPrettyPrint,
options.ExceptionHandler,
audit);
sourceRunnables.Add(runnable);
dataFlowSignalManager.AddSignalListener(operatorEntry.Key, runnable);
}
return new EPDataFlowInstanceImpl(
options.DataFlowInstanceUserObject,
options.DataFlowInstanceId,
statistics,
operators,
sourceRunnables,
dataflow,
agentInstanceContext,
statistics,
options.ParametersURIs);
}
private static object InstantiateOperator(
IContainer container,
int operatorNum,
DataflowDesc dataflow,
EPDataFlowInstantiationOptions options,
AgentInstanceContext agentInstanceContext)
{
var operatorFactory = dataflow.OperatorFactories.Get(operatorNum);
var metadata = dataflow.OperatorMetadata.Get(operatorNum);
// see if the operator is already provided by options
if (options.OperatorProvider != null) {
var operatorX = options.OperatorProvider.Provide(
new EPDataFlowOperatorProviderContext(
dataflow.DataflowName,
metadata.OperatorName,
operatorFactory));
if (operatorX != null) {
return operatorX;
}
}
IDictionary<string, object> additionalParameters = null;
if (options.ParametersURIs != null) {
var prefix = metadata.OperatorName + "/";
foreach (var entry in options.ParametersURIs) {
if (!entry.Key.StartsWith(prefix)) {
continue;
}
if (additionalParameters == null) {
additionalParameters = new Dictionary<string, object>();
}
additionalParameters.Put(entry.Key.Substring(prefix.Length), entry.Value);
}
}
object @operator;
try {
@operator = operatorFactory.Operator(
new DataFlowOpInitializeContext(
container,
dataflow.DataflowName,
metadata.OperatorName,
operatorNum,
agentInstanceContext,
additionalParameters,
options.DataFlowInstanceId,
options.ParameterProvider,
operatorFactory,
options.DataFlowInstanceUserObject));
}
catch (Exception t) {
var meta = dataflow.OperatorMetadata.Get(operatorNum);
throw new EPException(
"Failed to obtain operator instance for '" + meta.OperatorName + "': " + t.Message,
t);
}
return @operator;
}
public void Run(RegressionEnvironment env)
{
// one statement exists before the data flow
env.CompileDeploy("select Id from SupportBean_B");
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowOne " +
"create schema AllObjects as System.Object," +
"EPStatementSource -> thedata<AllObjects> {} " +
"DefaultSupportCaptureOp(thedata) {}");
var captureOp = new DefaultSupportCaptureOp(env.Container.LockManager());
var options = new EPDataFlowInstantiationOptions();
var myFilter = new MyFilter();
options.WithParameterProvider(
new DefaultSupportGraphParamProvider(
Collections.SingletonMap<string, object>("statementFilter", myFilter)));
options.WithOperatorProvider(new DefaultSupportGraphOpProvider(captureOp));
var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options);
df.Start();
env.SendEventBean(new SupportBean_B("B1"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "Id" },
new object[] {"B1"});
env.CompileDeploy("select TheString, IntPrimitive from SupportBean");
env.SendEventBean(new SupportBean("E1", 1));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "TheString","IntPrimitive" },
new object[] {"E1", 1});
env.CompileDeploy("@Name('s2') select Id from SupportBean_A");
env.SendEventBean(new SupportBean_A("A1"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "Id" },
new object[] {"A1"});
env.UndeployModuleContaining("s2");
env.SendEventBean(new SupportBean_A("A2"));
Sleep(50);
Assert.AreEqual(0, captureOp.Current.Length);
env.CompileDeploy("@Name('s2') select Id from SupportBean_A");
env.SendEventBean(new SupportBean_A("A3"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "Id" },
new object[] {"A3"});
env.SendEventBean(new SupportBean_B("B2"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(
env.Container,
captureOp.GetCurrentAndReset()[0],
new [] { "Id" },
new object[] {"B2"});
df.Cancel();
env.SendEventBean(new SupportBean("E1", 1));
env.SendEventBean(new SupportBean_A("A1"));
env.SendEventBean(new SupportBean_B("B3"));
Assert.AreEqual(0, captureOp.Current.Length);
env.UndeployAll();
}
public void Run(RegressionEnvironment env)
{
// declare
var path = new RegressionPath();
env.CompileDeploy("create schema SomeType ()", path);
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowOne " +
"DefaultSupportSourceOp -> outstream<SomeType> {name: 'SourceOne'}" +
"DefaultSupportSourceOp -> outstream<SomeType> {name: 'SourceTwo'}" +
"DefaultSupportCaptureOp(outstream) {}",
path);
// instantiate
var latchOne = new CountDownLatch(1);
var latchTwo = new CountDownLatch(1);
IDictionary<string, object> ops = new Dictionary<string, object>();
ops.Put(
"SourceOne",
new DefaultSupportSourceOp(
new object[] {
latchOne,
new object[] {1}
}));
ops.Put(
"SourceTwo",
new DefaultSupportSourceOp(
new object[] {
latchTwo,
new object[] {1}
}));
var future = new DefaultSupportCaptureOp(2, env.Container.LockManager());
ops.Put("DefaultSupportCaptureOp", future);
var options =
new EPDataFlowInstantiationOptions().WithOperatorProvider(
new DefaultSupportGraphOpProviderByOpName(ops));
var dfOne = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options);
try {
dfOne.Run();
Assert.Fail();
}
catch (UnsupportedOperationException ex) {
Assert.AreEqual(
"The data flow 'MyDataFlowOne' has zero or multiple sources and requires the use of the start method instead",
ex.Message);
}
latchTwo.CountDown();
dfOne.Start();
latchOne.CountDown();
try {
dfOne.Join();
}
catch (ThreadInterruptedException e) {
throw new EPException(e);
}
Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State);
Assert.AreEqual(2, future.GetAndReset().Count);
env.UndeployAll();
}
public void Run(RegressionEnvironment env)
{
// declare
var path = new RegressionPath();
env.CompileDeploy("create schema SomeType ()", path);
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowOne " +
"DefaultSupportSourceOp -> s<SomeType> {}" +
"DefaultSupportCaptureOp(s) {}",
path);
// instantiate
var latch = new CountDownLatch(1);
var source = new DefaultSupportSourceOp(
new object[] {
latch,
new object[] {1}
});
var future = new DefaultSupportCaptureOp(1, env.Container.LockManager());
var options =
new EPDataFlowInstantiationOptions().WithOperatorProvider(
new DefaultSupportGraphOpProvider(source, future));
var dfOne = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options);
Assert.AreEqual("MyDataFlowOne", dfOne.DataFlowName);
Assert.AreEqual(EPDataFlowState.INSTANTIATED, dfOne.State);
var joiningRunnable = new MyJoiningRunnable(dfOne);
var joiningThread = new Thread(joiningRunnable.Run);
joiningThread.Name = GetType().Name + "-joining";
var unlatchingThread = new Thread(
() => {
try {
while (dfOne.State != EPDataFlowState.RUNNING) {
Thread.Sleep(10);
}
Thread.Sleep(1000);
latch.CountDown();
}
catch (Exception e) {
log.Error("Unexpected exception", e);
}
});
unlatchingThread.Name = GetType().Name + "-unlatching";
joiningThread.Start();
unlatchingThread.Start();
dfOne.Run();
Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State);
Assert.AreEqual(1, future.GetAndReset()[0].Count);
Assert.AreEqual(2, source.CurrentCount);
try {
joiningThread.Join();
unlatchingThread.Join();
}
catch (ThreadInterruptedException e) {
throw new EPException(e);
}
var deltaJoin = joiningRunnable.End - joiningRunnable.Start;
Assert.IsTrue(deltaJoin >= 500, "deltaJoin=" + deltaJoin);
env.UndeployAll();
}
public void Run(RegressionEnvironment env)
{
foreach (var rep in new[] {EventRepresentationChoice.AVRO}) {
RunAssertionFields(env, rep, true);
RunAssertionFields(env, rep, false);
}
// test doc samples
var epl = "@Name('flow') create dataflow MyDataFlow\n" +
" create schema SampleSchema(tagId string, locX double, locY double)," +
" " +
" // BeaconSource that produces empty object-array events without delay or interval\n" +
" // until cancelled.\n" +
" BeaconSource -> stream.one {}\n" +
" \n" +
" // BeaconSource that produces one RFIDSchema event populating event properties\n" +
" // from a user-defined function \"generateTagId\" and values.\n" +
" BeaconSource -> stream.two<SampleSchema> {\n" +
" iterations : 1,\n" +
" tagId : generateTagId(),\n" +
" locX : 10,\n" +
" locY : 20 \n" +
" }\n" +
" \n" +
" // BeaconSource that produces 10 object-array events populating the Price property \n" +
" // with a random value.\n" +
" BeaconSource -> stream.three {\n" +
" iterations : 1,\n" +
" interval : 10, // every 10 seconds\n" +
" initialDelay : 5, // start after 5 seconds\n" +
" price : Randomizer.Random() * 100,\n" +
" }";
env.CompileDeploy(epl);
env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlow");
env.UndeployAll();
// test options-provided beacon field
var eplMinimal = "@Name('flow') create dataflow MyGraph " +
"BeaconSource -> outstream<SupportBean> {iterations:1} " +
"EventBusSink(outstream) {}";
env.CompileDeploy(eplMinimal);
var options = new EPDataFlowInstantiationOptions();
options.AddParameterURI("BeaconSource/TheString", "E1");
var instance = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyGraph", options);
env.CompileDeploy("@Name('s0') select * from SupportBean").AddListener("s0");
instance.Run();
Sleep(200);
EPAssertionUtil.AssertProps(
env.Listener("s0").AssertOneGetNewAndReset(),
new [] { "TheString" },
new object[] {"E1"});
// invalid: no output stream
TryInvalidCompile(
env,
"create dataflow DF1 BeaconSource {}",
"Failed to obtain operator 'BeaconSource': BeaconSource operator requires one output stream but produces 0 streams");
env.UndeployAll();
}
public void TestStatementFilter()
{
_epService.EPAdministrator.Configuration.AddEventType <SupportBean>();
_epService.EPAdministrator.Configuration.AddEventType(typeof(SupportBean_A));
_epService.EPAdministrator.Configuration.AddEventType(typeof(SupportBean_B));
// one statement exists before the data flow
var stmt = _epService.EPAdministrator.CreateEPL("select id from SupportBean_B");
_epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowOne " +
"create schema AllObjects Object," +
"EPStatementSource -> thedata<AllObjects> {} " +
"DefaultSupportCaptureOp(thedata) {}");
var captureOp = new DefaultSupportCaptureOp();
var options = new EPDataFlowInstantiationOptions();
var myFilter = new MyFilter();
options.ParameterProvider(new DefaultSupportGraphParamProvider(Collections.SingletonDataMap("statementFilter", myFilter)));
options.OperatorProvider(new DefaultSupportGraphOpProvider(captureOp));
var df = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options);
df.Start();
_epService.EPRuntime.SendEvent(new SupportBean_B("B1"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "B1" });
_epService.EPAdministrator.CreateEPL("select TheString, IntPrimitive from SupportBean");
_epService.EPRuntime.SendEvent(new SupportBean("E1", 1));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "TheString,IntPrimitive".Split(','), new Object[] { "E1", 1 });
var stmtTwo = _epService.EPAdministrator.CreateEPL("select id from SupportBean_A");
_epService.EPRuntime.SendEvent(new SupportBean_A("A1"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "A1" });
stmtTwo.Stop();
_epService.EPRuntime.SendEvent(new SupportBean_A("A2"));
Thread.Sleep(50);
Assert.AreEqual(0, captureOp.GetCurrent().Length);
stmtTwo.Start();
_epService.EPRuntime.SendEvent(new SupportBean_A("A3"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "A3" });
_epService.EPRuntime.SendEvent(new SupportBean_B("B2"));
captureOp.WaitForInvocation(200, 1);
EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "B2" });
df.Cancel();
_epService.EPRuntime.SendEvent(new SupportBean("E1", 1));
_epService.EPRuntime.SendEvent(new SupportBean_A("A1"));
_epService.EPRuntime.SendEvent(new SupportBean_B("B3"));
Assert.AreEqual(0, captureOp.GetCurrent().Length);
}
public static DataflowStartDesc Realize(String dataFlowName,
IDictionary <int, Object> operators,
IDictionary <int, OperatorMetadataDescriptor> operatorMetadata,
ICollection <int> operatorBuildOrder,
IList <LogicalChannelBinding> bindings,
DataFlowSignalManager dataFlowSignalManager,
EPDataFlowInstantiationOptions options,
EPServicesContext services,
StatementContext statementContext)
{
// First pass: inject runtime context
IDictionary <int, EPDataFlowEmitter> runtimeContexts = new Dictionary <int, EPDataFlowEmitter>();
OperatorStatisticsProvider statisticsProvider = null;
if (options.IsOperatorStatistics())
{
statisticsProvider = new OperatorStatisticsProvider(operatorMetadata);
}
bool audit = AuditEnum.DATAFLOW_OP.GetAudit(statementContext.Annotations) != null;
foreach (int producerOpNum in operatorBuildOrder)
{
String operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint;
if (Log.IsDebugEnabled)
{
Log.Debug("Generating runtime context for " + operatorPrettyPrint);
}
// determine the number of output streams
Object producingOp = operators.Get(producerOpNum);
int numOutputStreams = operatorMetadata.Get(producerOpNum).OperatorSpec.Output.Items.Count;
IList <ObjectBindingPair>[] targets = GetOperatorConsumersPerStream(
numOutputStreams, producerOpNum, operators, operatorMetadata, bindings);
EPDataFlowEmitter runtimeContext = GenerateRuntimeContext(
statementContext.EngineURI, statementContext.StatementName, audit, dataFlowName, producerOpNum,
operatorPrettyPrint, dataFlowSignalManager, targets, options);
if (options.IsOperatorStatistics())
{
runtimeContext = new EPDataFlowEmitterWrapperWStatistics(
runtimeContext, producerOpNum, statisticsProvider, options.IsCpuStatistics());
}
TypeHelper.SetFieldForAnnotation(producingOp, typeof(DataFlowContextAttribute), runtimeContext);
runtimeContexts.Put(producerOpNum, runtimeContext);
}
// Second pass: hook punctuation such that it gets forwarded
foreach (int producerOpNum in operatorBuildOrder)
{
String operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint;
if (Log.IsDebugEnabled)
{
Log.Debug("Handling signals for " + operatorPrettyPrint);
}
// determine consumers that receive punctuation
ICollection <int> consumingOperatorsWithPunctuation = new HashSet <int>();
foreach (LogicalChannelBinding binding in bindings)
{
if (!binding.LogicalChannel.OutputPort.HasPunctuation ||
binding.LogicalChannel.OutputPort.ProducingOpNum != producerOpNum)
{
continue;
}
consumingOperatorsWithPunctuation.Add(binding.LogicalChannel.ConsumingOpNum);
}
// hook up a listener for each
foreach (int consumerPunc in consumingOperatorsWithPunctuation)
{
EPDataFlowEmitter context = runtimeContexts.Get(consumerPunc);
if (context == null)
{
continue;
}
dataFlowSignalManager.AddSignalListener(
producerOpNum, new ProxyDataFlowSignalListener
{
ProcSignal = context.SubmitSignal
});
}
}
return(new DataflowStartDesc(statisticsProvider));
}
public void TestBeaconFields()
{
RunAssertionFields(EventRepresentationEnum.MAP, true);
RunAssertionFields(EventRepresentationEnum.OBJECTARRAY, true);
RunAssertionFields(EventRepresentationEnum.MAP, false);
RunAssertionFields(EventRepresentationEnum.OBJECTARRAY, false);
// test doc samples
_epService.EPAdministrator.Configuration.AddPlugInSingleRowFunction("GenerateTagId", GetType().FullName, "GenerateTagId");
var epl = "create dataflow MyDataFlow\n" +
" create schema SampleSchema(tagId string, locX double, locY double)," +
" " +
" // BeaconSource that produces empty object-array events without delay or interval\n" +
" // until cancelled.\n" +
" BeaconSource -> stream.one {}\n" +
" \n" +
" // BeaconSource that produces one RFIDSchema event populating event properties\n" +
" // from a user-defined function \"generateTagId\" and values.\n" +
" BeaconSource -> stream.two<SampleSchema> {\n" +
" iterations : 1,\n" +
" tagId : GenerateTagId(),\n" +
" locX : 10,\n" +
" locY : 20 \n" +
" }\n" +
" \n" +
" // BeaconSource that produces 10 object-array events populating the price property \n" +
" // with a random value.\n" +
" BeaconSource -> stream.three {\n" +
" iterations : 1,\n" +
" interval : 10, // every 10 seconds\n" +
" initialDelay : 5, // start after 5 seconds\n" +
" price : MyMath.Random() * 100,\n" +
" }";
_epService.EPAdministrator.CreateEPL(epl);
_epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlow");
// test options-provided beacon field
_epService.EPAdministrator.Configuration.AddEventType(typeof(SupportBean));
var eplMinimal = "create dataflow MyGraph " +
"BeaconSource -> outstream<SupportBean> {iterations:1} " +
"EventBusSink(outstream) {}";
_epService.EPAdministrator.CreateEPL(eplMinimal);
var options = new EPDataFlowInstantiationOptions();
options.AddParameterURI("BeaconSource/TheString", "E1");
var instance = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyGraph", options);
var listener = new SupportUpdateListener();
_epService.EPAdministrator.CreateEPL("select * from SupportBean").Events += listener.Update;
instance.Run();
Thread.Sleep(200);
EPAssertionUtil.AssertProps(listener.AssertOneGetNewAndReset(), "TheString".Split(','), new Object[] { "E1" });
// invalid: no output stream
SupportDataFlowAssertionUtil.TryInvalidInstantiate(_epService, "DF1", "create dataflow DF1 BeaconSource {}",
"Failed to instantiate data flow 'DF1': Failed initialization for operator 'BeaconSource': BeaconSource operator requires one output stream but produces 0 streams");
}
public void TestBlockingRunJoin()
{
// declare
_epService.EPAdministrator.CreateEPL(
"create dataflow MyDataFlowOne " +
"DefaultSupportSourceOp -> s<SomeType> {}" +
"DefaultSupportCaptureOp(s) {}");
// instantiate
var latch = new CountDownLatch(1);
var source = new DefaultSupportSourceOp(
new Object[]
{
latch, new Object[]
{
1
}
});
var future = new DefaultSupportCaptureOp(1);
EPDataFlowInstantiationOptions options =
new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(source, future));
EPDataFlowInstance dfOne = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options);
Assert.AreEqual("MyDataFlowOne", dfOne.DataFlowName);
Assert.AreEqual(EPDataFlowState.INSTANTIATED, dfOne.State);
var joiningRunnable = new MyJoiningRunnable(dfOne);
var joiningThread = new Thread(joiningRunnable.Run);
var unlatchingThread = new Thread(
() =>
{
try
{
while (dfOne.State != EPDataFlowState.RUNNING)
{
}
Thread.Sleep(1000);
latch.CountDown();
}
catch (Exception e)
{
Console.Error.WriteLine(e.StackTrace);
}
});
joiningThread.Start();
unlatchingThread.Start();
dfOne.Run();
Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State);
Assert.AreEqual(1, future.GetAndReset()[0].Count);
Assert.AreEqual(2, source.GetCurrentCount());
joiningThread.Join();
unlatchingThread.Join();
long deltaJoin = joiningRunnable.End - joiningRunnable.Start;
Assert.That(deltaJoin, Is.GreaterThanOrEqualTo(500));
}
public static OperatorStatisticsProvider Realize(
DataflowDesc dataflow,
IDictionary<int, object> operators,
IList<LogicalChannelBinding> bindings,
DataFlowSignalManager dataFlowSignalManager,
EPDataFlowInstantiationOptions options,
AgentInstanceContext agentInstanceContext)
{
IDictionary<int, OperatorMetadataDescriptor> operatorMetadata = dataflow.OperatorMetadata;
// First pass: inject runtime context
IDictionary<int, EPDataFlowEmitter> runtimeContexts = new Dictionary<int, EPDataFlowEmitter>();
OperatorStatisticsProvider statisticsProvider = null;
if (options.IsOperatorStatistics) {
statisticsProvider = new OperatorStatisticsProvider(operatorMetadata);
}
foreach (int producerOpNum in dataflow.OperatorBuildOrder) {
var operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint;
if (Log.IsDebugEnabled) {
Log.Debug("Generating runtime context for " + operatorPrettyPrint);
}
// determine the number of output streams
var producingOp = operators.Get(producerOpNum);
var numOutputStreams = operatorMetadata.Get(producerOpNum).NumOutputPorts;
var targets = GetOperatorConsumersPerStream(
numOutputStreams,
producerOpNum,
operators,
operatorMetadata,
bindings);
var runtimeContext = GenerateRuntimeContext(
agentInstanceContext,
dataflow,
options.DataFlowInstanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
targets,
options);
if (options.IsOperatorStatistics) {
runtimeContext = new EPDataFlowEmitterWrapperWStatistics(
runtimeContext,
producerOpNum,
statisticsProvider,
options.IsCpuStatistics);
}
TypeHelper.SetFieldForAnnotation(producingOp, typeof(DataFlowContextAttribute), runtimeContext);
runtimeContexts.Put(producerOpNum, runtimeContext);
}
// Second pass: hook punctuation such that it gets forwarded
foreach (int producerOpNum in dataflow.OperatorBuildOrder) {
var operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint;
if (Log.IsDebugEnabled) {
Log.Debug("Handling signals for " + operatorPrettyPrint);
}
// determine consumers that receive punctuation
ISet<int> consumingOperatorsWithPunctuation = new HashSet<int>();
foreach (var binding in bindings) {
if (!binding.LogicalChannel.OutputPort.HasPunctuation ||
binding.LogicalChannel.OutputPort.ProducingOpNum != producerOpNum) {
continue;
}
consumingOperatorsWithPunctuation.Add(binding.LogicalChannel.ConsumingOpNum);
}
// hook up a listener for each
foreach (var consumerPunc in consumingOperatorsWithPunctuation) {
var context = runtimeContexts.Get(consumerPunc);
if (context == null) {
continue;
}
dataFlowSignalManager.AddSignalListener(
producerOpNum,
new ProxyDataFlowSignalListener {
ProcProcessSignal = signal => context.SubmitSignal(signal)
});
}
}
return statisticsProvider;
}
private static void RunAssertionFields(
RegressionEnvironment env,
EventRepresentationChoice representationEnum,
bool eventbean)
{
EPDataFlowInstantiationOptions options;
var path = new RegressionPath();
var streamType = eventbean ? "EventBean<MyEvent>" : "MyEvent";
env.CompileDeploy(
representationEnum.GetAnnotationTextWJsonProvided<MyLocalJsonProvidedMyEvent>() +
"create schema MyEvent(p0 string, p1 long, p2 double)", path);
env.CompileDeploy(
"@Name('flow') create dataflow MyDataFlowOne " +
"" +
"BeaconSource -> BeaconStream<" + streamType + "> " +
"{" +
" iterations : 3," +
" p0 : 'abc'," +
" p1 : cast(Math.Round(Randomizer.Random() * 10) + 1, long)," +
" p2 : 1d," +
"}" +
"DefaultSupportCaptureOp(BeaconStream) {}",
path);
var future = new DefaultSupportCaptureOp(3, env.Container.LockManager());
options = new EPDataFlowInstantiationOptions()
.WithOperatorProvider(new DefaultSupportGraphOpProvider(future));
var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options);
df.Start();
object[] output;
try {
output = future.GetValue(2, TimeUnit.SECONDS);
}
catch (Exception t) {
throw new EPException(t);
}
Assert.AreEqual(3, output.Length);
for (var i = 0; i < 3; i++) {
if (!eventbean) {
if (representationEnum.IsObjectArrayEvent()) {
var row = (object[]) output[i];
Assert.AreEqual("abc", row[0]);
var val = row[1].AsInt64();
Assert.IsTrue(val >= 0 && val <= 11, "val=" + val);
Assert.AreEqual(1d, row[2]);
}
else if (representationEnum.IsMapEvent()) {
var row = (IDictionary<string, object>) output[i];
Assert.AreEqual("abc", row.Get("p0"));
var val = row.Get("p1").AsInt64();
Assert.IsTrue(val >= 0 && val <= 11, "val=" + val);
Assert.AreEqual(1d, row.Get("p2"));
}
else {
var row = (GenericRecord) output[i];
Assert.AreEqual("abc", row.Get("p0"));
var val = row.Get("p1").AsInt64();
Assert.IsTrue(val >= 0 && val <= 11, "val=" + val);
Assert.AreEqual(1d, row.Get("p2"));
}
}
else {
var row = (EventBean) output[i];
Assert.AreEqual("abc", row.Get("p0"));
}
}
env.UndeployAll();
}
