public void ConfigurationTests_Configuration_Filters()
{
var pipelineId = Guid.NewGuid().ToString();
var config = new PipelineConfiguration
{
Id = pipelineId,
InputHandler = new HandlerNode(typeof(GenericInputHandler <LogMessage>)),
OutputHandlers = new List <HandlerNode>
{
new HandlerNode("ConsoleOutput")
{
Filters = new List <string>
{
"Message -> msg"
}
}
}
};
var builder = new PipelineBuilder();
builder.BuildPipeline(ProcessingServer.Server, config);
var client = new EventDispatcher();
client.Process(pipelineId, new LogMessage {
Message = "ConfigurationTests_NewPipelinePerEvent1"
});
client.Process(pipelineId, new LogMessage {
Message = "ConfigurationTests_NewPipelinePerEvent2"
});
ProcessingServer.Server.WaitAll(pipelineId);
}
public void ConfigurationTests_Configuration()
{
var pipelineId = Guid.NewGuid().ToString();
var config = new PipelineConfiguration
{
Id = pipelineId,
InputHandler = new HandlerNode("CustomInput"),
OutputHandlers = new List <HandlerNode>
{
new HandlerNode("ConsoleOutput")
}
};
var builder = new PipelineBuilder();
builder.BuildPipeline(config);
var client = new EventDispatcher();
client.Process(pipelineId, new LogMessage {
Message = "ConfigurationTests_NewPipelinePerEvent1"
});
client.Process(pipelineId, new LogMessage {
Message = "ConfigurationTests_NewPipelinePerEvent2"
});
}
public void LoadTesting_MultipleProcessors()
{
GlobalConfiguration.Setup(s => s.UseOptions(new Options
{
MaxItemsInQueue = 10,
MinProcessors = 10,
MaxProcessors = 50
}));
var pipelineId = Guid.NewGuid().ToString();
var config = new PipelineConfiguration
{
Id = pipelineId,
InputHandler = new HandlerNode("CustomInput"),
OutputHandlers = new List <HandlerNode>
{
new HandlerNode("ConsoleOutput")
{
Filters = new List <string>
{
"Message -> msg",
"Level -> lvl"
}
},
new HandlerNode(typeof(ThreadWaitHandler))
}
};
var builder = new PipelineBuilder();
builder.BuildPipeline(config);
var result = ProfilerSession.StartSession()
.Task(ctx =>
{
var client = new EventDispatcher();
client.Process(pipelineId, new LogMessage {
Level = "Info", Message = $"Loadtesting {ctx.Get<int>(ContextKeys.Iteration)} on Thread {ctx.Get<int>(ContextKeys.ThreadId)}", Title = "Loadtest"
});
})
.SetIterations(100)
.SetThreads(10)
.Settings(s => s.RunWarmup = false)
.RunSession();
ProcessingServer.Server.WaitAll(pipelineId);
// delay to ensure all threads are ended
System.Threading.Tasks.Task.Delay(3000);
var storage = GlobalConfiguration.Configuration.Resolve <IStorage>();
Assert.GreaterOrEqual(100 * 10, storage.Get <long>(new StorageKey(pipelineId, "ProcessedEventsMetric")));
var stats = new StatisticsApi(pipelineId);
Assert.AreEqual(10, stats.GetMetricValue(MetricType.ThreadCount));
result.Trace("### LoadTesting");
}
public void PipelineBuilder_Resolver_Perfomance()
{
var server = new ProcessingServer();
var builder = new PipelineBuilder();
builder.BuildPipeline(server, new PipelineConfiguration
{
Id = "1",
InputHandler = new HandlerNode(typeof(PlainInputHandler)),
OutputHandlers = new List <HandlerNode>
{
new HandlerNode(typeof(PlainOutputHandler))
}
});
builder.BuildPipeline(server, new PipelineConfiguration
{
Id = "2",
InputHandler = new HandlerNode(typeof(CustomInputHandler)),
OutputHandlers = new List <HandlerNode>
{
new HandlerNode(typeof(PlainOutputHandler))
}
});
//server.EventBusFactory.GetEventBus("1").Publish(new Event("1", new EventData()));
//server.EventBusFactory.GetEventBus("2").Publish(new Event("2", new EventData()));
var runner = new BenchmarkRunner();
runner.SetIterations(10);
runner.AddSession("Simple resolver",
ProfilerSession.StartSession()
.Task(() => server.Publish(new Event("1", new EventData())))
.SetThreads(5)
);
runner.AddSession("Complex resolver",
ProfilerSession.StartSession()
.Task(() => server.Publish(new Event("2", new EventData())))
.SetThreads(5)
);
var result = runner.RunSessions();
result.Trace();
}
/// <summary>
/// creates or updates a pipeline based on the given configuration
/// </summary>
/// <param name="server"></param>
/// <param name="pipelineId"></param>
/// <param name="config"></param>
/// <returns></returns>
public static IProcessingServer SetupPipeline(this IProcessingServer server, string pipelineId, PipelineConfiguration config)
{
var builder = new PipelineBuilder();
builder.BuildPipeline(server, config);
return(server);
}
private static void BuildPipeline(Assembly source, String root, bool print)
{
PipelineBuilder tpb = new PipelineBuilder(source.Location);
List <PipelineSegmentSource> pipelines = tpb.BuildPipeline( );
foreach (PipelineSegmentSource pipline in pipelines)
{
String compRoot = root + "\\" + pipline.Name;
foreach (SourceFile file in pipline.Files)
{
}
}
}
public void PipelineBuilder_should_create_serializable_pipeline()
{
var context = new MLContext();
var columns = new Column[]
{
new Column("cat1", ColumnType.Catagorical, ColumnPurpose.CategoricalFeature),
new Column("numeric1", ColumnType.Numeric, ColumnPurpose.NumericFeature),
new Column("text1", ColumnType.Numeric, ColumnPurpose.TextFeature),
new Column("label", ColumnType.Catagorical, ColumnPurpose.Label),
};
var pb = new PipelineBuilder(TaskType.BinaryClassification, false, true);
var pipeline = pb.BuildPipeline(context, columns);
var pipelineDataContract = pipeline.ToDataContract();
var json = JsonConvert.SerializeObject(pipelineDataContract, Formatting.Indented);
Approvals.Verify(json);
var restoredPipeline = pipelineDataContract.ToPipeline(context);
var restoredJson = JsonConvert.SerializeObject(restoredPipeline.ToDataContract(), Formatting.Indented);
restoredJson.Should().Be(json);
}