private void RunAssertionNonBlockingJoinException(EPServiceProvider epService) { epService.EPAdministrator.CreateEPL("create schema SomeType ()"); epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowOne " + "DefaultSupportSourceOp -> outstream<SomeType> {}" + "DefaultSupportCaptureOp(outstream) {}"); var latchOne = new CountDownLatch(1); var src = new DefaultSupportSourceOp(new object[] {latchOne, new MyRuntimeException("TestException")}); var output = new DefaultSupportCaptureOp(SupportContainer.Instance.LockManager()); var options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(src, output)); var dfOne = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options); dfOne.Start(); var unlatchingThread = new Thread( () => { try { Thread.Sleep(300); latchOne.CountDown(); } catch (Exception e) { Console.Error.WriteLine(e.StackTrace); } }); unlatchingThread.Start(); dfOne.Join(); Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State); Assert.AreEqual(0, output.GetAndReset().Count); epService.EPAdministrator.DestroyAllStatements(); }
private void RunAssertionNonBlockingJoinMultipleRunnable(EPServiceProvider epService) { // declare epService.EPAdministrator.CreateEPL("create schema SomeType ()"); epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowOne " + "SourceOne -> outstream<SomeType> {}" + "SourceTwo -> outstream<SomeType> {}" + "Future(outstream) {}"); // instantiate var latchOne = new CountDownLatch(1); var latchTwo = new CountDownLatch(1); var ops = new Dictionary<string, object>(); ops.Put("SourceOne", new DefaultSupportSourceOp(new object[] {latchOne, new object[] {1}})); ops.Put("SourceTwo", new DefaultSupportSourceOp(new object[] {latchTwo, new object[] {1}})); var future = new DefaultSupportCaptureOp(2, SupportContainer.Instance.LockManager()); ops.Put("Future", future); var options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProviderByOpName(ops)); var dfOne = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options); dfOne.Start(); Thread.Sleep(50); Assert.AreEqual(EPDataFlowState.RUNNING, dfOne.State); latchOne.CountDown(); Thread.Sleep(200); Assert.AreEqual(EPDataFlowState.RUNNING, dfOne.State); latchTwo.CountDown(); dfOne.Join(); Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State); Assert.AreEqual(2, future.GetAndReset().Count); epService.EPAdministrator.DestroyAllStatements(); }
private void TrySource(Supplier <AdapterInputSource> sourceSupplier) { var source = sourceSupplier.Invoke(); var spec = new CSVInputAdapterSpec(source, "TypeC"); _runtime = EPRuntimeProvider.GetRuntime("testPlayFromInputStream", MakeConfig("TypeC")); _runtime.Initialize(); InputAdapter feed = new CSVInputAdapter(_runtime, spec); var stmt = CompileDeploy(_runtime, "select * from TypeC#length(100)").Statements[0]; var listener = new SupportUpdateListener(); stmt.Events += listener.Update; feed.Start(); Assert.AreEqual(1, listener.GetNewDataList().Count); source = sourceSupplier.Invoke(); // test graph var graph = "create dataflow ReadCSV " + "FileSource -> mystream<TypeC> { hasTitleLine: true }" + "DefaultSupportCaptureOp(mystream) {}"; var deployment = CompileDeploy(_runtime, graph); var outputOp = new DefaultSupportCaptureOp(); var options = new EPDataFlowInstantiationOptions(); options.OperatorProvider = new DefaultSupportGraphOpProvider(outputOp); options.ParameterProvider = new DefaultSupportGraphParamProvider(Collections.SingletonDataMap("adapterInputSource", source)); var instance = _runtime.DataFlowService.Instantiate(deployment.DeploymentId, "ReadCSV", options); instance.Run(); var received = outputOp.GetAndReset()[0].ToArray(); Assert.AreEqual(1, received.Length); }
private void TryAssertionJoinOrder(EPServiceProvider epService, string fromClause) { string graph = "create dataflow MySelect\n" + "Emitter -> instream_s0<SupportBean_S0>{name: 'emitterS0'}\n" + "Emitter -> instream_s1<SupportBean_S1>{name: 'emitterS1'}\n" + "Emitter -> instream_s2<SupportBean_S2>{name: 'emitterS2'}\n" + "Select(instream_s0 as S0, instream_s1 as S1, instream_s2 as S2) -> outstream {\n" + " select: (select s0.id as s0id, s1.id as s1id, s2.id as s2id " + fromClause + ")\n" + "}\n" + "CaptureOp(outstream) {}\n"; EPStatement stmtGraph = epService.EPAdministrator.CreateEPL(graph); var capture = new DefaultSupportCaptureOp(SupportContainer.Instance.LockManager()); IDictionary <string, Object> operators = CollectionUtil.PopulateNameValueMap("CaptureOp", capture); var options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProviderByOpName(operators)); EPDataFlowInstance instance = epService.EPRuntime.DataFlowRuntime.Instantiate("MySelect", options); EPDataFlowInstanceCaptive captive = instance.StartCaptive(); captive.Emitters.Get("emitterS0").Submit(new SupportBean_S0(1)); captive.Emitters.Get("emitterS1").Submit(new SupportBean_S1(10)); Assert.AreEqual(0, capture.Current.Length); captive.Emitters.Get("emitterS2").Submit(new SupportBean_S2(100)); Assert.AreEqual(1, capture.Current.Length); EPAssertionUtil.AssertProps( epService.Container, capture.GetCurrentAndReset()[0], "s0id,s1id,s2id".Split(','), new object[] { 1, 10, 100 }); instance.Cancel(); captive.Emitters.Get("emitterS2").Submit(new SupportBean_S2(101)); Assert.AreEqual(0, capture.Current.Length); stmtGraph.Dispose(); }
private void RunAssertionAllTypes(String typeName, Object[] events) { String graph = "create dataflow MySelect\n" + "DefaultSupportSourceOp -> instream<" + typeName + ">{}\n" + "Select(instream as ME) -> outstream {select: (select myString, sum(myInt) as total from ME)}\n" + "DefaultSupportCaptureOp(outstream) {}"; EPStatement stmtGraph = _epService.EPAdministrator.CreateEPL(graph); DefaultSupportSourceOp source = new DefaultSupportSourceOp(events); DefaultSupportCaptureOp capture = new DefaultSupportCaptureOp(2); EPDataFlowInstantiationOptions options = new EPDataFlowInstantiationOptions(); options.OperatorProvider(new DefaultSupportGraphOpProvider(source, capture)); EPDataFlowInstance instance = _epService.EPRuntime.DataFlowRuntime.Instantiate("MySelect", options); instance.Run(); Object[] result = capture.GetAndReset()[0].ToArray(); EPAssertionUtil.AssertPropsPerRow(result, "myString,total".Split(','), new Object[][] { new Object[] { "one", 1 }, new Object[] { "two", 3 } }); instance.Cancel(); stmtGraph.Dispose(); }
private void RunAssertionSendEventDynamicType(EPServiceProvider epService) { epService.EPAdministrator.CreateEPL("create objectarray schema MyEventOne(type string, p0 int, p1 string)"); epService.EPAdministrator.CreateEPL("create objectarray schema MyEventTwo(type string, f0 string, f1 int)"); var listener = new SupportUpdateListener(); epService.EPAdministrator.CreateEPL("select * from MyEventOne").Events += listener.Update; var listenerTwo = new SupportUpdateListener(); epService.EPAdministrator.CreateEPL("select * from MyEventTwo").Events += listenerTwo.Update; epService.EPAdministrator.CreateEPL("create dataflow MyDataFlow " + "MyObjectArrayGraphSource -> OutStream<?> {}" + "EventBusSink(OutStream) {" + " collector : {" + " class: '" + typeof(MyTransformToEventBus).FullName + "'" + " }" + "}"); var source = new MyObjectArrayGraphSource(Collections.List( new object[] { "type1", 100, "abc" }, new object[] { "type2", "GE", -1 } ).GetEnumerator()); var options = new EPDataFlowInstantiationOptions() .OperatorProvider(new DefaultSupportGraphOpProvider(source)); epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlow", options).Start(); listener.WaitForInvocation(3000, 1); listenerTwo.WaitForInvocation(3000, 1); EPAssertionUtil.AssertProps(listener.AssertOneGetNewAndReset(), "p0,p1".Split(','), new object[] { 100, "abc" }); EPAssertionUtil.AssertProps(listenerTwo.AssertOneGetNewAndReset(), "f0,f1".Split(','), new object[] { "GE", -1 }); epService.EPAdministrator.DestroyAllStatements(); }
public void SaveConfiguration( string dataflowConfigName, string deploymentId, string dataFlowName, EPDataFlowInstantiationOptions options) { lock (this) { var entry = GetEntryMayNull(deploymentId, dataFlowName); if (entry == null) { var message = "Failed to locate data flow '" + dataFlowName + "'"; throw new EPDataFlowNotFoundException(message); } if (_configurationState.Exists(dataflowConfigName)) { var message = "Data flow saved configuration by name '" + dataflowConfigName + "' already exists"; throw new EPDataFlowAlreadyExistsException(message); } _configurationState.Add( new EPDataFlowSavedConfiguration(dataflowConfigName, deploymentId, dataFlowName, options)); } }
public void TestIterateFinalMarker() { _epService.EPAdministrator.Configuration.AddEventType <SupportBean>(); String graph = "create dataflow MySelect\n" + "Emitter -> instream_s0<SupportBean>{name: 'emitterS0'}\n" + "@Audit Select(instream_s0 as ALIAS) -> outstream {\n" + " select: (select TheString, sum(IntPrimitive) as SumInt from ALIAS group by TheString order by TheString asc),\n" + " iterate: true" + "}\n" + "CaptureOp(outstream) {}\n"; _epService.EPRuntime.SendEvent(new CurrentTimeEvent(0)); _epService.EPAdministrator.CreateEPL(graph); DefaultSupportCaptureOp capture = new DefaultSupportCaptureOp(); IDictionary <String, Object> operators = CollectionUtil.PopulateNameValueMap("CaptureOp", capture); EPDataFlowInstantiationOptions options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProviderByOpName(operators)); EPDataFlowInstance instance = _epService.EPRuntime.DataFlowRuntime.Instantiate("MySelect", options); EPDataFlowInstanceCaptive captive = instance.StartCaptive(); Emitter emitter = captive.Emitters.Get("emitterS0"); emitter.Submit(new SupportBean("E3", 4)); emitter.Submit(new SupportBean("E2", 3)); emitter.Submit(new SupportBean("E1", 1)); emitter.Submit(new SupportBean("E2", 2)); emitter.Submit(new SupportBean("E1", 5)); Assert.AreEqual(0, capture.GetCurrent().Length); emitter.SubmitSignal(new EPDataFlowSignalFinalMarkerImpl()); EPAssertionUtil.AssertPropsPerRow(capture.GetCurrent(), "TheString,SumInt".Split(','), new Object[][] { new Object[] { "E1", 6 }, new Object[] { "E2", 5 }, new Object[] { "E3", 4 } }); instance.Cancel(); }
public void Run(RegressionEnvironment env) { // declare env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowOne " + "BeaconSource -> BeaconStream {iterations : 1}" + "DefaultSupportCaptureOp(BeaconStream) {}"); // instantiate var future = new DefaultSupportCaptureOp(1, env.Container.LockManager()); var options = new EPDataFlowInstantiationOptions().WithOperatorProvider( new DefaultSupportGraphOpProvider(future)); var dfOne = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options); Assert.AreEqual("MyDataFlowOne", dfOne.DataFlowName); Assert.AreEqual(EPDataFlowState.INSTANTIATED, dfOne.State); // has not run Sleep(1000); Assert.IsFalse(future.IsDone()); // blocking run dfOne.Run(); Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State); try { Assert.AreEqual(1, future.GetValueOrDefault().Length); } catch (Exception t) { throw new EPException(t); } // assert past-exec TryAssertionAfterExec(dfOne); env.UndeployAll(); }
private void RunAssertionBeaconNoType(EPServiceProvider epService) { EPDataFlowInstantiationOptions options; object[] output; epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowOne " + "BeaconSource -> BeaconStream {}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var countExpected = 10; var futureAtLeast = new DefaultSupportCaptureOp(countExpected, SupportContainer.Instance.LockManager()); options = new EPDataFlowInstantiationOptions() .OperatorProvider(new DefaultSupportGraphOpProvider(futureAtLeast)); var df = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options); df.Start(); output = futureAtLeast.GetValue(1, TimeUnit.SECONDS); Assert.IsTrue(countExpected <= output.Length); df.Cancel(); // BeaconSource with given number of iterations epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowTwo " + "BeaconSource -> BeaconStream {" + " iterations: 5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureExactTwo = new DefaultSupportCaptureOp(5, SupportContainer.Instance.LockManager()); options = new EPDataFlowInstantiationOptions() .OperatorProvider(new DefaultSupportGraphOpProvider(futureExactTwo)); epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowTwo", options).Start(); output = futureExactTwo.GetValue(1, TimeUnit.SECONDS); Assert.AreEqual(5, output.Length); // BeaconSource with delay epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowThree " + "BeaconSource -> BeaconStream {" + " iterations: 2," + " initialDelay: 0.5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureExactThree = new DefaultSupportCaptureOp(2, SupportContainer.Instance.LockManager()); options = new EPDataFlowInstantiationOptions() .OperatorProvider(new DefaultSupportGraphOpProvider(futureExactThree)); var start = PerformanceObserver.MilliTime; epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowThree", options).Start(); output = futureExactThree.GetValue(1, TimeUnit.SECONDS); var end = PerformanceObserver.MilliTime; Assert.AreEqual(2, output.Length); Assert.IsTrue(end - start > 490, "delta=" + (end - start)); // BeaconSource with period epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowFour " + "BeaconSource -> BeaconStream {" + " interval: 0.5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureFour = new DefaultSupportCaptureOp(2, SupportContainer.Instance.LockManager()); options = new EPDataFlowInstantiationOptions() .OperatorProvider(new DefaultSupportGraphOpProvider(futureFour)); epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFour", options).Start(); output = futureFour.GetValue(1, TimeUnit.SECONDS); Assert.AreEqual(2, output.Length); // test Beacon with define typed epService.EPAdministrator.CreateEPL("create objectarray schema MyTestOAType(p1 string)"); epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowFive " + "BeaconSource -> BeaconStream<MyTestOAType> {" + " interval: 0.5," + " p1 : 'abc'" + "}"); epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFive"); }
private void RunAssertionFields( EPServiceProvider epService, EventRepresentationChoice representationEnum, bool eventbean) { EPDataFlowInstantiationOptions options; epService.EPAdministrator.CreateEPL( "create " + representationEnum.GetOutputTypeCreateSchemaName() + " schema MyEvent(p0 string, p1 long, p2 double)"); var stmtGraph = epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowOne " + "" + "BeaconSource -> BeaconStream<" + (eventbean ? "EventBean<MyEvent>" : "MyEvent") + "> {" + " iterations : 3," + " p0 : 'abc'," + " p1 : MyMath.Round(MyMath.Random() * 10) + 1," + " p2 : 1d," + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var future = new DefaultSupportCaptureOp(3, SupportContainer.Instance.LockManager()); options = new EPDataFlowInstantiationOptions() .OperatorProvider(new DefaultSupportGraphOpProvider(future)); var df = epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options); df.Start(); var output = future.GetValue(2, TimeUnit.SECONDS); Assert.AreEqual(3, output.Length); for (var i = 0; i < 3; i++) { if (!eventbean) { if (representationEnum.IsObjectArrayEvent()) { var row = (object[])output[i]; Assert.AreEqual("abc", row[0]); var val = (long)row[1]; Assert.IsTrue(val >= 0 && val <= 11, "val=" + val); Assert.AreEqual(1d, row[2]); } else if (representationEnum.IsMapEvent()) { Map row = (Map)output[i]; Assert.AreEqual("abc", row.Get("p0")); var val = (long)row.Get("p1"); Assert.IsTrue(val >= 0 && val <= 11, "val=" + val); Assert.AreEqual(1d, row.Get("p2")); } else { var row = (GenericRecord)output[i]; Assert.AreEqual("abc", row.Get("p0")); var val = (long)row.Get("p1"); Assert.IsTrue(val >= 0 && val <= 11, "val=" + val); Assert.AreEqual(1d, row.Get("p2")); } } else { var row = (EventBean)output[i]; Assert.AreEqual("abc", row.Get("p0")); } } stmtGraph.Dispose(); epService.EPAdministrator.Configuration.RemoveEventType("MyEvent", true); }
public void Run(RegressionEnvironment env) { var fields = new [] { "p0","p1" }; env.CompileDeploy( "@Name('flow') create dataflow MyDataFlow " + "Emitter -> outstream<MyOAEventType> {name:'src1'}" + "DefaultSupportCaptureOp(outstream) {}"); var captureOp = new DefaultSupportCaptureOp(); var options = new EPDataFlowInstantiationOptions(); options.WithOperatorProvider(new DefaultSupportGraphOpProvider(captureOp)); var instance = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlow", options); var captiveStart = instance.StartCaptive(); Assert.AreEqual(0, captiveStart.Runnables.Count); Assert.AreEqual(1, captiveStart.Emitters.Count); var emitter = captiveStart.Emitters.Get("src1"); Assert.AreEqual(EPDataFlowState.RUNNING, instance.State); emitter.Submit(new object[] {"E1", 10}); EPAssertionUtil.AssertPropsPerRow( env.Container, captureOp.Current, fields, new[] {new object[] {"E1", 10}}); emitter.Submit(new object[] {"E2", 20}); EPAssertionUtil.AssertPropsPerRow( env.Container, captureOp.Current, fields, new[] {new object[] {"E1", 10}, new object[] {"E2", 20}}); emitter.SubmitSignal(new EPDataFlowSignalFinalMarkerImpl()); EPAssertionUtil.AssertPropsPerRow( env.Container, captureOp.Current, fields, new object[0][]); EPAssertionUtil.AssertPropsPerRow( env.Container, captureOp.GetAndReset()[0].UnwrapIntoArray<object>(), fields, new[] {new object[] {"E1", 10}, new object[] {"E2", 20}}); emitter.Submit(new object[] {"E3", 30}); EPAssertionUtil.AssertPropsPerRow( env.Container, captureOp.Current, fields, new[] {new object[] {"E3", 30}}); // stays running until cancelled (no transition to complete) Assert.AreEqual(EPDataFlowState.RUNNING, instance.State); instance.Cancel(); Assert.AreEqual(EPDataFlowState.CANCELLED, instance.State); env.UndeployAll(); // test doc sample var epl = "@Name('flow') create dataflow HelloWorldDataFlow\n" + " create schema SampleSchema(text string),\t// sample type\t\t\n" + "\t\n" + " Emitter -> helloworld.stream<SampleSchema> { name: 'myemitter' }\n" + " LogSink(helloworld.stream) {}"; env.CompileDeploy(epl); env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "HelloWorldDataFlow"); env.UndeployAll(); }
public void Run(RegressionEnvironment env) { env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowOne " + "create map schema SingleProp (Id string), " + "EPStatementSource -> thedata<SingleProp> {" + " statementDeploymentId : 'MyDeploymentId'," + " statementName : 'MyStatement'," + "} " + "DefaultSupportCaptureOp(thedata) {}"); var captureOp = new DefaultSupportCaptureOp(); var options = new EPDataFlowInstantiationOptions() .WithOperatorProvider(new DefaultSupportGraphOpProvider(captureOp)); var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options); Assert.IsNull(df.UserObject); Assert.IsNull(df.InstanceId); df.Start(); env.SendEventBean(new SupportBean("E1", 1)); Assert.AreEqual(0, captureOp.Current.Length); var epl = "@Name('MyStatement') select TheString as Id from SupportBean"; var compiled = env.Compile(epl); try { env.Deployment.Deploy(compiled, new DeploymentOptions().WithDeploymentId("MyDeploymentId")); } catch (EPDeployException e) { Assert.Fail(e.Message); } env.SendEventBean(new SupportBean("E2", 2)); captureOp.WaitForInvocation(100, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "Id" }, new object[] {"E2"}); env.UndeployModuleContaining("MyStatement"); env.SendEventBean(new SupportBean("E3", 3)); Assert.AreEqual(0, captureOp.Current.Length); try { env.Deployment.Deploy(compiled, new DeploymentOptions().WithDeploymentId("MyDeploymentId")); } catch (EPDeployException e) { Assert.Fail(e.Message); } env.SendEventBean(new SupportBean("E4", 4)); captureOp.WaitForInvocation(100, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "Id" }, new object[] {"E4"}); env.UndeployModuleContaining("MyStatement"); env.SendEventBean(new SupportBean("E5", 5)); Assert.AreEqual(0, captureOp.Current.Length); compiled = env.Compile("@Name('MyStatement') select 'X'||TheString||'X' as Id from SupportBean"); try { env.Deployment.Deploy(compiled, new DeploymentOptions().WithDeploymentId("MyDeploymentId")); } catch (EPDeployException e) { Assert.Fail(e.Message); } env.SendEventBean(new SupportBean("E6", 6)); captureOp.WaitForInvocation(100, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "Id" }, new object[] {"XE6X"}); df.Cancel(); env.UndeployAll(); }
public void TestBeaconNoType() { EPDataFlowInstantiationOptions options; Object[] output; _epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowOne " + "BeaconSource -> BeaconStream {}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var countExpected = 10; var futureAtLeast = new DefaultSupportCaptureOp(countExpected); options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureAtLeast)); var df = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options); df.Start(); output = futureAtLeast.GetValue(TimeSpan.FromSeconds(1)); Assert.IsTrue(countExpected <= output.Length); df.Cancel(); // BeaconSource with given number of iterations _epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowTwo " + "BeaconSource -> BeaconStream {" + " iterations: 5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureExactTwo = new DefaultSupportCaptureOp(5); options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureExactTwo)); _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowTwo", options).Start(); output = futureExactTwo.GetValue(TimeSpan.FromSeconds(1)); Assert.AreEqual(5, output.Length); // BeaconSource with delay _epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowThree " + "BeaconSource -> BeaconStream {" + " iterations: 2," + " initialDelay: 0.5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureExactThree = new DefaultSupportCaptureOp(2); options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureExactThree)); long start = Environment.TickCount; _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowThree", options).Start(); output = futureExactThree.GetValue(TimeSpan.FromSeconds(1)); long end = Environment.TickCount; Assert.AreEqual(2, output.Length); Assert.IsTrue(end - start > 490, "delta=" + (end - start)); // BeaconSource with period _epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowFour " + "BeaconSource -> BeaconStream {" + " interval: 0.5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureFour = new DefaultSupportCaptureOp(2); options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(futureFour)); _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFour", options).Start(); output = futureFour.GetValue(TimeSpan.FromSeconds(1)); Assert.AreEqual(2, output.Length); // test Beacon with define typed _epService.EPAdministrator.CreateEPL("create objectarray schema MyTestOAType(p1 string)"); _epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowFive " + "BeaconSource -> BeaconStream<MyTestOAType> {" + " interval: 0.5," + " p1 : 'abc'" + "}"); _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowFive"); }
private static EPDataFlowEmitter GenerateRuntimeContext( AgentInstanceContext agentInstanceContext, DataflowDesc dataflow, string instanceId, int producerOpNum, string operatorPrettyPrint, DataFlowSignalManager dataFlowSignalManager, IList<ObjectBindingPair>[] targetsPerStream, EPDataFlowInstantiationOptions options) { // handle no targets if (targetsPerStream == null) { return new EPDataFlowEmitterNoTarget(producerOpNum, dataFlowSignalManager); } var dataflowName = dataflow.DataflowName; var classpathImportService = agentInstanceContext.ImportServiceRuntime; // handle single-stream case if (targetsPerStream.Length == 1) { var targets = targetsPerStream[0]; // handle single-stream single target case if (targets.Count == 1) { var target = targets[0]; return GetSubmitHandler( agentInstanceContext, dataflow.DataflowName, instanceId, producerOpNum, operatorPrettyPrint, dataFlowSignalManager, target, options.ExceptionHandler, classpathImportService); } var handlers = new SubmitHandler[targets.Count]; for (var i = 0; i < handlers.Length; i++) { handlers[i] = GetSubmitHandler( agentInstanceContext, dataflowName, instanceId, producerOpNum, operatorPrettyPrint, dataFlowSignalManager, targets[i], options.ExceptionHandler, classpathImportService); } return new EPDataFlowEmitter1StreamNTarget(producerOpNum, dataFlowSignalManager, handlers); } // handle multi-stream case var handlersPerStream = new SubmitHandler[targetsPerStream.Length][]; for (var streamNum = 0; streamNum < targetsPerStream.Length; streamNum++) { var handlers = new SubmitHandler[targetsPerStream[streamNum].Count]; handlersPerStream[streamNum] = handlers; for (var i = 0; i < handlers.Length; i++) { handlers[i] = GetSubmitHandler( agentInstanceContext, dataflowName, instanceId, producerOpNum, operatorPrettyPrint, dataFlowSignalManager, targetsPerStream[streamNum][i], options.ExceptionHandler, classpathImportService); } } return new EPDataFlowEmitterNStreamNTarget(producerOpNum, dataFlowSignalManager, handlersPerStream); }
public void Run(RegressionEnvironment env) { SupportGraphSource.GetAndResetLifecycle(); var compiled = env.Compile( "@Name('flow') create dataflow MyDataFlow @Name('Goodie') @Audit SupportGraphSource -> outstream<SupportBean> {propOne:'abc'}"); var events = SupportGraphSourceForge.GetAndResetLifecycle(); Assert.AreEqual(3, events.Count); Assert.AreEqual("instantiated", events[0]); Assert.AreEqual("SetPropOne=abc", events[1]); var forgeCtx = (DataFlowOpForgeInitializeContext) events[2]; Assert.AreEqual(0, forgeCtx.InputPorts.Count); Assert.AreEqual(1, forgeCtx.OutputPorts.Count); Assert.AreEqual("outstream", forgeCtx.OutputPorts[0].StreamName); Assert.AreEqual("SupportBean", forgeCtx.OutputPorts[0].OptionalDeclaredType.EventType.Name); Assert.AreEqual(2, forgeCtx.OperatorAnnotations.Length); Assert.AreEqual("Goodie", ((NameAttribute) forgeCtx.OperatorAnnotations[0]).Value); Assert.IsNotNull((AuditAttribute) forgeCtx.OperatorAnnotations[1]); Assert.AreEqual("MyDataFlow", forgeCtx.DataflowName); Assert.AreEqual(0, forgeCtx.OperatorNumber); env.Deploy(compiled); events = SupportGraphSourceFactory.GetAndResetLifecycle(); Assert.AreEqual(3, events.Count); Assert.AreEqual("instantiated", events[0]); Assert.AreEqual("SetPropOne=abc", events[1]); var factoryCtx = (DataFlowOpFactoryInitializeContext) events[2]; Assert.AreEqual("MyDataFlow", factoryCtx.DataFlowName); Assert.AreEqual(0, factoryCtx.OperatorNumber); Assert.IsNotNull(factoryCtx.StatementContext); // instantiate var options = new EPDataFlowInstantiationOptions() .WithDataFlowInstanceId("id1") .WithDataFlowInstanceUserObject("myobject"); var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlow", options); events = SupportGraphSourceFactory.GetAndResetLifecycle(); Assert.AreEqual(1, events.Count); var opCtx = (DataFlowOpInitializeContext) events[0]; Assert.AreEqual("MyDataFlow", opCtx.DataFlowName); Assert.AreEqual("id1", opCtx.DataFlowInstanceId); Assert.IsNotNull(opCtx.AgentInstanceContext); Assert.AreEqual("myobject", opCtx.DataflowInstanceUserObject); Assert.AreEqual(0, opCtx.OperatorNumber); Assert.AreEqual("SupportGraphSource", opCtx.OperatorName); events = SupportGraphSource.GetAndResetLifecycle(); Assert.AreEqual(1, events.Count); Assert.AreEqual("instantiated", events[0]); // instantiated // run df.Run(); events = SupportGraphSource.GetAndResetLifecycle(); Assert.AreEqual(5, events.Count); Assert.IsTrue(events[0] is DataFlowOpOpenContext); // called open (GraphSource only) Assert.AreEqual("next(numrows=0)", events[1]); Assert.AreEqual("next(numrows=1)", events[2]); Assert.AreEqual("next(numrows=2)", events[3]); Assert.IsTrue(events[4] is DataFlowOpCloseContext); // called close (GraphSource only) env.UndeployAll(); }
public void Run(RegressionEnvironment env) { EPDataFlowInstantiationOptions options; object[] output; env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowOne " + "BeaconSource -> BeaconStream {}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var countExpected = 10; var futureAtLeast = new DefaultSupportCaptureOp(countExpected, env.Container.LockManager()); options = new EPDataFlowInstantiationOptions() .WithOperatorProvider(new DefaultSupportGraphOpProvider(futureAtLeast)); var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options); df.Start(); try { output = futureAtLeast.GetValue(1, TimeUnit.SECONDS); } catch (Exception e) { throw new EPException(e); } Assert.IsTrue(countExpected <= output.Length); df.Cancel(); env.UndeployAll(); // BeaconSource with given number of iterations env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowTwo " + "BeaconSource -> BeaconStream {" + " iterations: 5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureExactTwo = new DefaultSupportCaptureOp(5, env.Container.LockManager()); options = new EPDataFlowInstantiationOptions() .WithOperatorProvider(new DefaultSupportGraphOpProvider(futureExactTwo)); env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowTwo", options).Start(); try { output = futureExactTwo.GetValue(1, TimeUnit.SECONDS); } catch (Exception t) { throw new EPException(t); } Assert.AreEqual(5, output.Length); env.UndeployAll(); // BeaconSource with delay env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowThree " + "BeaconSource -> BeaconStream {" + " iterations: 2," + " initialDelay: 0.5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureExactThree = new DefaultSupportCaptureOp(2, env.Container.LockManager()); options = new EPDataFlowInstantiationOptions() .WithOperatorProvider(new DefaultSupportGraphOpProvider(futureExactThree)); var start = PerformanceObserver.MilliTime; env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowThree", options).Start(); try { output = futureExactThree.GetValue(1, TimeUnit.SECONDS); } catch (Exception e) { throw new EPException(e); } var end = PerformanceObserver.MilliTime; Assert.AreEqual(2, output.Length); Assert.IsTrue(end - start > 490, "delta=" + (end - start)); env.UndeployAll(); // BeaconSource with period env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowFour " + "BeaconSource -> BeaconStream {" + " interval: 0.5" + "}" + "DefaultSupportCaptureOp(BeaconStream) {}"); var futureFour = new DefaultSupportCaptureOp(2, env.Container.LockManager()); options = new EPDataFlowInstantiationOptions() .WithOperatorProvider(new DefaultSupportGraphOpProvider(futureFour)); var instance = env.Runtime.DataFlowService.Instantiate( env.DeploymentId("flow"), "MyDataFlowFour", options); instance.Start(); try { output = futureFour.GetValue(2, TimeUnit.SECONDS); } catch (Exception t) { throw new EPException(t); } Assert.AreEqual(2, output.Length); instance.Cancel(); env.UndeployAll(); // test Beacon with define typed var path = new RegressionPath(); env.CompileDeploy("create objectarray schema MyTestOAType(p1 string)", path); env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowFive " + "BeaconSource -> BeaconStream<MyTestOAType> {" + " interval: 0.5," + " p1 : 'abc'" + "}", path); env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowFive"); env.UndeployAll(); }
public static EPDataFlowInstance Instantiate( IContainer container, int agentInstanceId, DataflowDesc dataflow, EPDataFlowInstantiationOptions options) { var statementContext = dataflow.StatementContext; // allocate agent instance context var @lock = statementContext.StatementAgentInstanceLockFactory.GetStatementLock( statementContext.StatementName, statementContext.Annotations, statementContext.IsStatelessSelect, statementContext.StatementType); var handle = new EPStatementAgentInstanceHandle(statementContext.EpStatementHandle, agentInstanceId, @lock); var auditProvider = statementContext.StatementInformationals.AuditProvider; var instrumentationProvider = statementContext.StatementInformationals.InstrumentationProvider; var agentInstanceContext = new AgentInstanceContext(statementContext, handle, null, null, auditProvider, instrumentationProvider); // assure variables statementContext.VariableManagementService.SetLocalVersion(); // instantiate operators var operators = InstantiateOperators(container, agentInstanceContext, options, dataflow); // determine binding of each channel to input methods (ports) IList<LogicalChannelBinding> operatorChannelBindings = new List<LogicalChannelBinding>(); foreach (var channel in dataflow.LogicalChannels) { var targetClass = operators.Get(channel.ConsumingOpNum).GetType(); var consumingMethod = FindMatchingMethod(channel.ConsumingOpPrettyPrint, targetClass, channel, false); LogicalChannelBindingMethodDesc onSignalMethod = null; if (channel.OutputPort.HasPunctuation) { onSignalMethod = FindMatchingMethod(channel.ConsumingOpPrettyPrint, targetClass, channel, true); } operatorChannelBindings.Add(new LogicalChannelBinding(channel, consumingMethod, onSignalMethod)); } // obtain realization var dataFlowSignalManager = new DataFlowSignalManager(); var statistics = DataflowInstantiatorHelper.Realize( dataflow, operators, operatorChannelBindings, dataFlowSignalManager, options, agentInstanceContext); // For each GraphSource add runnable IList<GraphSourceRunnable> sourceRunnables = new List<GraphSourceRunnable>(); var audit = AuditEnum.DATAFLOW_SOURCE.GetAudit(statementContext.Annotations) != null; foreach (var operatorEntry in operators) { if (!(operatorEntry.Value is DataFlowSourceOperator)) { continue; } var meta = dataflow.OperatorMetadata.Get(operatorEntry.Key); var graphSource = (DataFlowSourceOperator) operatorEntry.Value; var runnable = new GraphSourceRunnable( agentInstanceContext, graphSource, dataflow.DataflowName, options.DataFlowInstanceId, meta.OperatorName, operatorEntry.Key, meta.OperatorPrettyPrint, options.ExceptionHandler, audit); sourceRunnables.Add(runnable); dataFlowSignalManager.AddSignalListener(operatorEntry.Key, runnable); } return new EPDataFlowInstanceImpl( options.DataFlowInstanceUserObject, options.DataFlowInstanceId, statistics, operators, sourceRunnables, dataflow, agentInstanceContext, statistics, options.ParametersURIs); }
private static object InstantiateOperator( IContainer container, int operatorNum, DataflowDesc dataflow, EPDataFlowInstantiationOptions options, AgentInstanceContext agentInstanceContext) { var operatorFactory = dataflow.OperatorFactories.Get(operatorNum); var metadata = dataflow.OperatorMetadata.Get(operatorNum); // see if the operator is already provided by options if (options.OperatorProvider != null) { var operatorX = options.OperatorProvider.Provide( new EPDataFlowOperatorProviderContext( dataflow.DataflowName, metadata.OperatorName, operatorFactory)); if (operatorX != null) { return operatorX; } } IDictionary<string, object> additionalParameters = null; if (options.ParametersURIs != null) { var prefix = metadata.OperatorName + "/"; foreach (var entry in options.ParametersURIs) { if (!entry.Key.StartsWith(prefix)) { continue; } if (additionalParameters == null) { additionalParameters = new Dictionary<string, object>(); } additionalParameters.Put(entry.Key.Substring(prefix.Length), entry.Value); } } object @operator; try { @operator = operatorFactory.Operator( new DataFlowOpInitializeContext( container, dataflow.DataflowName, metadata.OperatorName, operatorNum, agentInstanceContext, additionalParameters, options.DataFlowInstanceId, options.ParameterProvider, operatorFactory, options.DataFlowInstanceUserObject)); } catch (Exception t) { var meta = dataflow.OperatorMetadata.Get(operatorNum); throw new EPException( "Failed to obtain operator instance for '" + meta.OperatorName + "': " + t.Message, t); } return @operator; }
public void Run(RegressionEnvironment env) { // one statement exists before the data flow env.CompileDeploy("select Id from SupportBean_B"); env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowOne " + "create schema AllObjects as System.Object," + "EPStatementSource -> thedata<AllObjects> {} " + "DefaultSupportCaptureOp(thedata) {}"); var captureOp = new DefaultSupportCaptureOp(env.Container.LockManager()); var options = new EPDataFlowInstantiationOptions(); var myFilter = new MyFilter(); options.WithParameterProvider( new DefaultSupportGraphParamProvider( Collections.SingletonMap<string, object>("statementFilter", myFilter))); options.WithOperatorProvider(new DefaultSupportGraphOpProvider(captureOp)); var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options); df.Start(); env.SendEventBean(new SupportBean_B("B1")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "Id" }, new object[] {"B1"}); env.CompileDeploy("select TheString, IntPrimitive from SupportBean"); env.SendEventBean(new SupportBean("E1", 1)); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "TheString","IntPrimitive" }, new object[] {"E1", 1}); env.CompileDeploy("@Name('s2') select Id from SupportBean_A"); env.SendEventBean(new SupportBean_A("A1")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "Id" }, new object[] {"A1"}); env.UndeployModuleContaining("s2"); env.SendEventBean(new SupportBean_A("A2")); Sleep(50); Assert.AreEqual(0, captureOp.Current.Length); env.CompileDeploy("@Name('s2') select Id from SupportBean_A"); env.SendEventBean(new SupportBean_A("A3")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "Id" }, new object[] {"A3"}); env.SendEventBean(new SupportBean_B("B2")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps( env.Container, captureOp.GetCurrentAndReset()[0], new [] { "Id" }, new object[] {"B2"}); df.Cancel(); env.SendEventBean(new SupportBean("E1", 1)); env.SendEventBean(new SupportBean_A("A1")); env.SendEventBean(new SupportBean_B("B3")); Assert.AreEqual(0, captureOp.Current.Length); env.UndeployAll(); }
public void Run(RegressionEnvironment env) { // declare var path = new RegressionPath(); env.CompileDeploy("create schema SomeType ()", path); env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowOne " + "DefaultSupportSourceOp -> outstream<SomeType> {name: 'SourceOne'}" + "DefaultSupportSourceOp -> outstream<SomeType> {name: 'SourceTwo'}" + "DefaultSupportCaptureOp(outstream) {}", path); // instantiate var latchOne = new CountDownLatch(1); var latchTwo = new CountDownLatch(1); IDictionary<string, object> ops = new Dictionary<string, object>(); ops.Put( "SourceOne", new DefaultSupportSourceOp( new object[] { latchOne, new object[] {1} })); ops.Put( "SourceTwo", new DefaultSupportSourceOp( new object[] { latchTwo, new object[] {1} })); var future = new DefaultSupportCaptureOp(2, env.Container.LockManager()); ops.Put("DefaultSupportCaptureOp", future); var options = new EPDataFlowInstantiationOptions().WithOperatorProvider( new DefaultSupportGraphOpProviderByOpName(ops)); var dfOne = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options); try { dfOne.Run(); Assert.Fail(); } catch (UnsupportedOperationException ex) { Assert.AreEqual( "The data flow 'MyDataFlowOne' has zero or multiple sources and requires the use of the start method instead", ex.Message); } latchTwo.CountDown(); dfOne.Start(); latchOne.CountDown(); try { dfOne.Join(); } catch (ThreadInterruptedException e) { throw new EPException(e); } Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State); Assert.AreEqual(2, future.GetAndReset().Count); env.UndeployAll(); }
public void Run(RegressionEnvironment env) { // declare var path = new RegressionPath(); env.CompileDeploy("create schema SomeType ()", path); env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowOne " + "DefaultSupportSourceOp -> s<SomeType> {}" + "DefaultSupportCaptureOp(s) {}", path); // instantiate var latch = new CountDownLatch(1); var source = new DefaultSupportSourceOp( new object[] { latch, new object[] {1} }); var future = new DefaultSupportCaptureOp(1, env.Container.LockManager()); var options = new EPDataFlowInstantiationOptions().WithOperatorProvider( new DefaultSupportGraphOpProvider(source, future)); var dfOne = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options); Assert.AreEqual("MyDataFlowOne", dfOne.DataFlowName); Assert.AreEqual(EPDataFlowState.INSTANTIATED, dfOne.State); var joiningRunnable = new MyJoiningRunnable(dfOne); var joiningThread = new Thread(joiningRunnable.Run); joiningThread.Name = GetType().Name + "-joining"; var unlatchingThread = new Thread( () => { try { while (dfOne.State != EPDataFlowState.RUNNING) { Thread.Sleep(10); } Thread.Sleep(1000); latch.CountDown(); } catch (Exception e) { log.Error("Unexpected exception", e); } }); unlatchingThread.Name = GetType().Name + "-unlatching"; joiningThread.Start(); unlatchingThread.Start(); dfOne.Run(); Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State); Assert.AreEqual(1, future.GetAndReset()[0].Count); Assert.AreEqual(2, source.CurrentCount); try { joiningThread.Join(); unlatchingThread.Join(); } catch (ThreadInterruptedException e) { throw new EPException(e); } var deltaJoin = joiningRunnable.End - joiningRunnable.Start; Assert.IsTrue(deltaJoin >= 500, "deltaJoin=" + deltaJoin); env.UndeployAll(); }
public void Run(RegressionEnvironment env) { foreach (var rep in new[] {EventRepresentationChoice.AVRO}) { RunAssertionFields(env, rep, true); RunAssertionFields(env, rep, false); } // test doc samples var epl = "@Name('flow') create dataflow MyDataFlow\n" + " create schema SampleSchema(tagId string, locX double, locY double)," + " " + " // BeaconSource that produces empty object-array events without delay or interval\n" + " // until cancelled.\n" + " BeaconSource -> stream.one {}\n" + " \n" + " // BeaconSource that produces one RFIDSchema event populating event properties\n" + " // from a user-defined function \"generateTagId\" and values.\n" + " BeaconSource -> stream.two<SampleSchema> {\n" + " iterations : 1,\n" + " tagId : generateTagId(),\n" + " locX : 10,\n" + " locY : 20 \n" + " }\n" + " \n" + " // BeaconSource that produces 10 object-array events populating the Price property \n" + " // with a random value.\n" + " BeaconSource -> stream.three {\n" + " iterations : 1,\n" + " interval : 10, // every 10 seconds\n" + " initialDelay : 5, // start after 5 seconds\n" + " price : Randomizer.Random() * 100,\n" + " }"; env.CompileDeploy(epl); env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlow"); env.UndeployAll(); // test options-provided beacon field var eplMinimal = "@Name('flow') create dataflow MyGraph " + "BeaconSource -> outstream<SupportBean> {iterations:1} " + "EventBusSink(outstream) {}"; env.CompileDeploy(eplMinimal); var options = new EPDataFlowInstantiationOptions(); options.AddParameterURI("BeaconSource/TheString", "E1"); var instance = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyGraph", options); env.CompileDeploy("@Name('s0') select * from SupportBean").AddListener("s0"); instance.Run(); Sleep(200); EPAssertionUtil.AssertProps( env.Listener("s0").AssertOneGetNewAndReset(), new [] { "TheString" }, new object[] {"E1"}); // invalid: no output stream TryInvalidCompile( env, "create dataflow DF1 BeaconSource {}", "Failed to obtain operator 'BeaconSource': BeaconSource operator requires one output stream but produces 0 streams"); env.UndeployAll(); }
public void TestStatementFilter() { _epService.EPAdministrator.Configuration.AddEventType <SupportBean>(); _epService.EPAdministrator.Configuration.AddEventType(typeof(SupportBean_A)); _epService.EPAdministrator.Configuration.AddEventType(typeof(SupportBean_B)); // one statement exists before the data flow var stmt = _epService.EPAdministrator.CreateEPL("select id from SupportBean_B"); _epService.EPAdministrator.CreateEPL("create dataflow MyDataFlowOne " + "create schema AllObjects Object," + "EPStatementSource -> thedata<AllObjects> {} " + "DefaultSupportCaptureOp(thedata) {}"); var captureOp = new DefaultSupportCaptureOp(); var options = new EPDataFlowInstantiationOptions(); var myFilter = new MyFilter(); options.ParameterProvider(new DefaultSupportGraphParamProvider(Collections.SingletonDataMap("statementFilter", myFilter))); options.OperatorProvider(new DefaultSupportGraphOpProvider(captureOp)); var df = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options); df.Start(); _epService.EPRuntime.SendEvent(new SupportBean_B("B1")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "B1" }); _epService.EPAdministrator.CreateEPL("select TheString, IntPrimitive from SupportBean"); _epService.EPRuntime.SendEvent(new SupportBean("E1", 1)); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "TheString,IntPrimitive".Split(','), new Object[] { "E1", 1 }); var stmtTwo = _epService.EPAdministrator.CreateEPL("select id from SupportBean_A"); _epService.EPRuntime.SendEvent(new SupportBean_A("A1")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "A1" }); stmtTwo.Stop(); _epService.EPRuntime.SendEvent(new SupportBean_A("A2")); Thread.Sleep(50); Assert.AreEqual(0, captureOp.GetCurrent().Length); stmtTwo.Start(); _epService.EPRuntime.SendEvent(new SupportBean_A("A3")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "A3" }); _epService.EPRuntime.SendEvent(new SupportBean_B("B2")); captureOp.WaitForInvocation(200, 1); EPAssertionUtil.AssertProps(captureOp.GetCurrentAndReset()[0], "id".Split(','), new Object[] { "B2" }); df.Cancel(); _epService.EPRuntime.SendEvent(new SupportBean("E1", 1)); _epService.EPRuntime.SendEvent(new SupportBean_A("A1")); _epService.EPRuntime.SendEvent(new SupportBean_B("B3")); Assert.AreEqual(0, captureOp.GetCurrent().Length); }
public static DataflowStartDesc Realize(String dataFlowName, IDictionary <int, Object> operators, IDictionary <int, OperatorMetadataDescriptor> operatorMetadata, ICollection <int> operatorBuildOrder, IList <LogicalChannelBinding> bindings, DataFlowSignalManager dataFlowSignalManager, EPDataFlowInstantiationOptions options, EPServicesContext services, StatementContext statementContext) { // First pass: inject runtime context IDictionary <int, EPDataFlowEmitter> runtimeContexts = new Dictionary <int, EPDataFlowEmitter>(); OperatorStatisticsProvider statisticsProvider = null; if (options.IsOperatorStatistics()) { statisticsProvider = new OperatorStatisticsProvider(operatorMetadata); } bool audit = AuditEnum.DATAFLOW_OP.GetAudit(statementContext.Annotations) != null; foreach (int producerOpNum in operatorBuildOrder) { String operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint; if (Log.IsDebugEnabled) { Log.Debug("Generating runtime context for " + operatorPrettyPrint); } // determine the number of output streams Object producingOp = operators.Get(producerOpNum); int numOutputStreams = operatorMetadata.Get(producerOpNum).OperatorSpec.Output.Items.Count; IList <ObjectBindingPair>[] targets = GetOperatorConsumersPerStream( numOutputStreams, producerOpNum, operators, operatorMetadata, bindings); EPDataFlowEmitter runtimeContext = GenerateRuntimeContext( statementContext.EngineURI, statementContext.StatementName, audit, dataFlowName, producerOpNum, operatorPrettyPrint, dataFlowSignalManager, targets, options); if (options.IsOperatorStatistics()) { runtimeContext = new EPDataFlowEmitterWrapperWStatistics( runtimeContext, producerOpNum, statisticsProvider, options.IsCpuStatistics()); } TypeHelper.SetFieldForAnnotation(producingOp, typeof(DataFlowContextAttribute), runtimeContext); runtimeContexts.Put(producerOpNum, runtimeContext); } // Second pass: hook punctuation such that it gets forwarded foreach (int producerOpNum in operatorBuildOrder) { String operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint; if (Log.IsDebugEnabled) { Log.Debug("Handling signals for " + operatorPrettyPrint); } // determine consumers that receive punctuation ICollection <int> consumingOperatorsWithPunctuation = new HashSet <int>(); foreach (LogicalChannelBinding binding in bindings) { if (!binding.LogicalChannel.OutputPort.HasPunctuation || binding.LogicalChannel.OutputPort.ProducingOpNum != producerOpNum) { continue; } consumingOperatorsWithPunctuation.Add(binding.LogicalChannel.ConsumingOpNum); } // hook up a listener for each foreach (int consumerPunc in consumingOperatorsWithPunctuation) { EPDataFlowEmitter context = runtimeContexts.Get(consumerPunc); if (context == null) { continue; } dataFlowSignalManager.AddSignalListener( producerOpNum, new ProxyDataFlowSignalListener { ProcSignal = context.SubmitSignal }); } } return(new DataflowStartDesc(statisticsProvider)); }
public void TestBeaconFields() { RunAssertionFields(EventRepresentationEnum.MAP, true); RunAssertionFields(EventRepresentationEnum.OBJECTARRAY, true); RunAssertionFields(EventRepresentationEnum.MAP, false); RunAssertionFields(EventRepresentationEnum.OBJECTARRAY, false); // test doc samples _epService.EPAdministrator.Configuration.AddPlugInSingleRowFunction("GenerateTagId", GetType().FullName, "GenerateTagId"); var epl = "create dataflow MyDataFlow\n" + " create schema SampleSchema(tagId string, locX double, locY double)," + " " + " // BeaconSource that produces empty object-array events without delay or interval\n" + " // until cancelled.\n" + " BeaconSource -> stream.one {}\n" + " \n" + " // BeaconSource that produces one RFIDSchema event populating event properties\n" + " // from a user-defined function \"generateTagId\" and values.\n" + " BeaconSource -> stream.two<SampleSchema> {\n" + " iterations : 1,\n" + " tagId : GenerateTagId(),\n" + " locX : 10,\n" + " locY : 20 \n" + " }\n" + " \n" + " // BeaconSource that produces 10 object-array events populating the price property \n" + " // with a random value.\n" + " BeaconSource -> stream.three {\n" + " iterations : 1,\n" + " interval : 10, // every 10 seconds\n" + " initialDelay : 5, // start after 5 seconds\n" + " price : MyMath.Random() * 100,\n" + " }"; _epService.EPAdministrator.CreateEPL(epl); _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlow"); // test options-provided beacon field _epService.EPAdministrator.Configuration.AddEventType(typeof(SupportBean)); var eplMinimal = "create dataflow MyGraph " + "BeaconSource -> outstream<SupportBean> {iterations:1} " + "EventBusSink(outstream) {}"; _epService.EPAdministrator.CreateEPL(eplMinimal); var options = new EPDataFlowInstantiationOptions(); options.AddParameterURI("BeaconSource/TheString", "E1"); var instance = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyGraph", options); var listener = new SupportUpdateListener(); _epService.EPAdministrator.CreateEPL("select * from SupportBean").Events += listener.Update; instance.Run(); Thread.Sleep(200); EPAssertionUtil.AssertProps(listener.AssertOneGetNewAndReset(), "TheString".Split(','), new Object[] { "E1" }); // invalid: no output stream SupportDataFlowAssertionUtil.TryInvalidInstantiate(_epService, "DF1", "create dataflow DF1 BeaconSource {}", "Failed to instantiate data flow 'DF1': Failed initialization for operator 'BeaconSource': BeaconSource operator requires one output stream but produces 0 streams"); }
public void TestBlockingRunJoin() { // declare _epService.EPAdministrator.CreateEPL( "create dataflow MyDataFlowOne " + "DefaultSupportSourceOp -> s<SomeType> {}" + "DefaultSupportCaptureOp(s) {}"); // instantiate var latch = new CountDownLatch(1); var source = new DefaultSupportSourceOp( new Object[] { latch, new Object[] { 1 } }); var future = new DefaultSupportCaptureOp(1); EPDataFlowInstantiationOptions options = new EPDataFlowInstantiationOptions().OperatorProvider(new DefaultSupportGraphOpProvider(source, future)); EPDataFlowInstance dfOne = _epService.EPRuntime.DataFlowRuntime.Instantiate("MyDataFlowOne", options); Assert.AreEqual("MyDataFlowOne", dfOne.DataFlowName); Assert.AreEqual(EPDataFlowState.INSTANTIATED, dfOne.State); var joiningRunnable = new MyJoiningRunnable(dfOne); var joiningThread = new Thread(joiningRunnable.Run); var unlatchingThread = new Thread( () => { try { while (dfOne.State != EPDataFlowState.RUNNING) { } Thread.Sleep(1000); latch.CountDown(); } catch (Exception e) { Console.Error.WriteLine(e.StackTrace); } }); joiningThread.Start(); unlatchingThread.Start(); dfOne.Run(); Assert.AreEqual(EPDataFlowState.COMPLETE, dfOne.State); Assert.AreEqual(1, future.GetAndReset()[0].Count); Assert.AreEqual(2, source.GetCurrentCount()); joiningThread.Join(); unlatchingThread.Join(); long deltaJoin = joiningRunnable.End - joiningRunnable.Start; Assert.That(deltaJoin, Is.GreaterThanOrEqualTo(500)); }
public static OperatorStatisticsProvider Realize( DataflowDesc dataflow, IDictionary<int, object> operators, IList<LogicalChannelBinding> bindings, DataFlowSignalManager dataFlowSignalManager, EPDataFlowInstantiationOptions options, AgentInstanceContext agentInstanceContext) { IDictionary<int, OperatorMetadataDescriptor> operatorMetadata = dataflow.OperatorMetadata; // First pass: inject runtime context IDictionary<int, EPDataFlowEmitter> runtimeContexts = new Dictionary<int, EPDataFlowEmitter>(); OperatorStatisticsProvider statisticsProvider = null; if (options.IsOperatorStatistics) { statisticsProvider = new OperatorStatisticsProvider(operatorMetadata); } foreach (int producerOpNum in dataflow.OperatorBuildOrder) { var operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint; if (Log.IsDebugEnabled) { Log.Debug("Generating runtime context for " + operatorPrettyPrint); } // determine the number of output streams var producingOp = operators.Get(producerOpNum); var numOutputStreams = operatorMetadata.Get(producerOpNum).NumOutputPorts; var targets = GetOperatorConsumersPerStream( numOutputStreams, producerOpNum, operators, operatorMetadata, bindings); var runtimeContext = GenerateRuntimeContext( agentInstanceContext, dataflow, options.DataFlowInstanceId, producerOpNum, operatorPrettyPrint, dataFlowSignalManager, targets, options); if (options.IsOperatorStatistics) { runtimeContext = new EPDataFlowEmitterWrapperWStatistics( runtimeContext, producerOpNum, statisticsProvider, options.IsCpuStatistics); } TypeHelper.SetFieldForAnnotation(producingOp, typeof(DataFlowContextAttribute), runtimeContext); runtimeContexts.Put(producerOpNum, runtimeContext); } // Second pass: hook punctuation such that it gets forwarded foreach (int producerOpNum in dataflow.OperatorBuildOrder) { var operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint; if (Log.IsDebugEnabled) { Log.Debug("Handling signals for " + operatorPrettyPrint); } // determine consumers that receive punctuation ISet<int> consumingOperatorsWithPunctuation = new HashSet<int>(); foreach (var binding in bindings) { if (!binding.LogicalChannel.OutputPort.HasPunctuation || binding.LogicalChannel.OutputPort.ProducingOpNum != producerOpNum) { continue; } consumingOperatorsWithPunctuation.Add(binding.LogicalChannel.ConsumingOpNum); } // hook up a listener for each foreach (var consumerPunc in consumingOperatorsWithPunctuation) { var context = runtimeContexts.Get(consumerPunc); if (context == null) { continue; } dataFlowSignalManager.AddSignalListener( producerOpNum, new ProxyDataFlowSignalListener { ProcProcessSignal = signal => context.SubmitSignal(signal) }); } } return statisticsProvider; }
private static void RunAssertionFields( RegressionEnvironment env, EventRepresentationChoice representationEnum, bool eventbean) { EPDataFlowInstantiationOptions options; var path = new RegressionPath(); var streamType = eventbean ? "EventBean<MyEvent>" : "MyEvent"; env.CompileDeploy( representationEnum.GetAnnotationTextWJsonProvided<MyLocalJsonProvidedMyEvent>() + "create schema MyEvent(p0 string, p1 long, p2 double)", path); env.CompileDeploy( "@Name('flow') create dataflow MyDataFlowOne " + "" + "BeaconSource -> BeaconStream<" + streamType + "> " + "{" + " iterations : 3," + " p0 : 'abc'," + " p1 : cast(Math.Round(Randomizer.Random() * 10) + 1, long)," + " p2 : 1d," + "}" + "DefaultSupportCaptureOp(BeaconStream) {}", path); var future = new DefaultSupportCaptureOp(3, env.Container.LockManager()); options = new EPDataFlowInstantiationOptions() .WithOperatorProvider(new DefaultSupportGraphOpProvider(future)); var df = env.Runtime.DataFlowService.Instantiate(env.DeploymentId("flow"), "MyDataFlowOne", options); df.Start(); object[] output; try { output = future.GetValue(2, TimeUnit.SECONDS); } catch (Exception t) { throw new EPException(t); } Assert.AreEqual(3, output.Length); for (var i = 0; i < 3; i++) { if (!eventbean) { if (representationEnum.IsObjectArrayEvent()) { var row = (object[]) output[i]; Assert.AreEqual("abc", row[0]); var val = row[1].AsInt64(); Assert.IsTrue(val >= 0 && val <= 11, "val=" + val); Assert.AreEqual(1d, row[2]); } else if (representationEnum.IsMapEvent()) { var row = (IDictionary<string, object>) output[i]; Assert.AreEqual("abc", row.Get("p0")); var val = row.Get("p1").AsInt64(); Assert.IsTrue(val >= 0 && val <= 11, "val=" + val); Assert.AreEqual(1d, row.Get("p2")); } else { var row = (GenericRecord) output[i]; Assert.AreEqual("abc", row.Get("p0")); var val = row.Get("p1").AsInt64(); Assert.IsTrue(val >= 0 && val <= 11, "val=" + val); Assert.AreEqual(1d, row.Get("p2")); } } else { var row = (EventBean) output[i]; Assert.AreEqual("abc", row.Get("p0")); } } env.UndeployAll(); }