public void ConfigurationTests_Configuration_Filters() { var pipelineId = Guid.NewGuid().ToString(); var config = new PipelineConfiguration { Id = pipelineId, InputHandler = new HandlerNode(typeof(GenericInputHandler <LogMessage>)), OutputHandlers = new List <HandlerNode> { new HandlerNode("ConsoleOutput") { Filters = new List <string> { "Message -> msg" } } } }; var builder = new PipelineBuilder(); builder.BuildPipeline(ProcessingServer.Server, config); var client = new EventDispatcher(); client.Process(pipelineId, new LogMessage { Message = "ConfigurationTests_NewPipelinePerEvent1" }); client.Process(pipelineId, new LogMessage { Message = "ConfigurationTests_NewPipelinePerEvent2" }); ProcessingServer.Server.WaitAll(pipelineId); }
public void ConfigurationTests_Configuration() { var pipelineId = Guid.NewGuid().ToString(); var config = new PipelineConfiguration { Id = pipelineId, InputHandler = new HandlerNode("CustomInput"), OutputHandlers = new List <HandlerNode> { new HandlerNode("ConsoleOutput") } }; var builder = new PipelineBuilder(); builder.BuildPipeline(config); var client = new EventDispatcher(); client.Process(pipelineId, new LogMessage { Message = "ConfigurationTests_NewPipelinePerEvent1" }); client.Process(pipelineId, new LogMessage { Message = "ConfigurationTests_NewPipelinePerEvent2" }); }
public void LoadTesting_MultipleProcessors() { GlobalConfiguration.Setup(s => s.UseOptions(new Options { MaxItemsInQueue = 10, MinProcessors = 10, MaxProcessors = 50 })); var pipelineId = Guid.NewGuid().ToString(); var config = new PipelineConfiguration { Id = pipelineId, InputHandler = new HandlerNode("CustomInput"), OutputHandlers = new List <HandlerNode> { new HandlerNode("ConsoleOutput") { Filters = new List <string> { "Message -> msg", "Level -> lvl" } }, new HandlerNode(typeof(ThreadWaitHandler)) } }; var builder = new PipelineBuilder(); builder.BuildPipeline(config); var result = ProfilerSession.StartSession() .Task(ctx => { var client = new EventDispatcher(); client.Process(pipelineId, new LogMessage { Level = "Info", Message = $"Loadtesting {ctx.Get<int>(ContextKeys.Iteration)} on Thread {ctx.Get<int>(ContextKeys.ThreadId)}", Title = "Loadtest" }); }) .SetIterations(100) .SetThreads(10) .Settings(s => s.RunWarmup = false) .RunSession(); ProcessingServer.Server.WaitAll(pipelineId); // delay to ensure all threads are ended System.Threading.Tasks.Task.Delay(3000); var storage = GlobalConfiguration.Configuration.Resolve <IStorage>(); Assert.GreaterOrEqual(100 * 10, storage.Get <long>(new StorageKey(pipelineId, "ProcessedEventsMetric"))); var stats = new StatisticsApi(pipelineId); Assert.AreEqual(10, stats.GetMetricValue(MetricType.ThreadCount)); result.Trace("### LoadTesting"); }
public void PipelineBuilder_Resolver_Perfomance() { var server = new ProcessingServer(); var builder = new PipelineBuilder(); builder.BuildPipeline(server, new PipelineConfiguration { Id = "1", InputHandler = new HandlerNode(typeof(PlainInputHandler)), OutputHandlers = new List <HandlerNode> { new HandlerNode(typeof(PlainOutputHandler)) } }); builder.BuildPipeline(server, new PipelineConfiguration { Id = "2", InputHandler = new HandlerNode(typeof(CustomInputHandler)), OutputHandlers = new List <HandlerNode> { new HandlerNode(typeof(PlainOutputHandler)) } }); //server.EventBusFactory.GetEventBus("1").Publish(new Event("1", new EventData())); //server.EventBusFactory.GetEventBus("2").Publish(new Event("2", new EventData())); var runner = new BenchmarkRunner(); runner.SetIterations(10); runner.AddSession("Simple resolver", ProfilerSession.StartSession() .Task(() => server.Publish(new Event("1", new EventData()))) .SetThreads(5) ); runner.AddSession("Complex resolver", ProfilerSession.StartSession() .Task(() => server.Publish(new Event("2", new EventData()))) .SetThreads(5) ); var result = runner.RunSessions(); result.Trace(); }
/// <summary> /// creates or updates a pipeline based on the given configuration /// </summary> /// <param name="server"></param> /// <param name="pipelineId"></param> /// <param name="config"></param> /// <returns></returns> public static IProcessingServer SetupPipeline(this IProcessingServer server, string pipelineId, PipelineConfiguration config) { var builder = new PipelineBuilder(); builder.BuildPipeline(server, config); return(server); }
private static void BuildPipeline(Assembly source, String root, bool print) { PipelineBuilder tpb = new PipelineBuilder(source.Location); List <PipelineSegmentSource> pipelines = tpb.BuildPipeline( ); foreach (PipelineSegmentSource pipline in pipelines) { String compRoot = root + "\\" + pipline.Name; foreach (SourceFile file in pipline.Files) { } } }
public void PipelineBuilder_should_create_serializable_pipeline() { var context = new MLContext(); var columns = new Column[] { new Column("cat1", ColumnType.Catagorical, ColumnPurpose.CategoricalFeature), new Column("numeric1", ColumnType.Numeric, ColumnPurpose.NumericFeature), new Column("text1", ColumnType.Numeric, ColumnPurpose.TextFeature), new Column("label", ColumnType.Catagorical, ColumnPurpose.Label), }; var pb = new PipelineBuilder(TaskType.BinaryClassification, false, true); var pipeline = pb.BuildPipeline(context, columns); var pipelineDataContract = pipeline.ToDataContract(); var json = JsonConvert.SerializeObject(pipelineDataContract, Formatting.Indented); Approvals.Verify(json); var restoredPipeline = pipelineDataContract.ToPipeline(context); var restoredJson = JsonConvert.SerializeObject(restoredPipeline.ToDataContract(), Formatting.Indented); restoredJson.Should().Be(json); }