public void TestReadPage()
{
var connection = new Connection {
Name = "input",
Provider = "elasticsearch",
Index = "colors",
Server = "localhost",
Port = 9200
}.WithValidation();
connection.Url = connection.BuildElasticUrl();
var pool = new SingleNodeConnectionPool(new Uri(connection.Url));
var settings = new ConnectionConfiguration(pool);
var client = new ElasticLowLevelClient(settings);
var context = new ConnectionContext(new PipelineContext(new TraceLogger(), null, new Entity {
Name = "rows",
Alias = "rows",
Page = 2,
PageSize = 20
}.WithDefaults()), connection);
var code = new Field {
Name = "code", Index = 0
}.WithDefaults();
var total = new Field {
Name = "total", Type = "int", Index = 1
}.WithDefaults();
var reader = new ElasticReader(context, new[] { code, total }, client, new RowFactory(2, false, false), ReadFrom.Input);
var rows = reader.Read().ToArray();
Assert.AreEqual(20, rows.Length);
}
private static void ReadDataFromElastic()
{
Console.WriteLine("");
Console.WriteLine("=======================================");
Console.WriteLine("");
Console.WriteLine("starting to read data from Elastic");
Console.WriteLine("");
var reader = new ElasticReader();
reader.ListAllIndexes();
reader.CountAllTypes();
var firstUser = reader.GetUserId();
// reader.UserStats(firstUser);
reader.EachUserWinnings();
reader.ReadComplexBetsQuery();
}
protected override void Load(ContainerBuilder builder)
{
if (_process == null)
{
return;
}
//CONNECTIONS
foreach (var connection in _process.Connections.Where(c => c.Provider.In("elasticsearch")))
{
connection.Url = connection.BuildElasticUrl();
builder.Register <IConnectionPool>(ctx => new SingleNodeConnectionPool(new Uri(connection.Url))).Named <IConnectionPool>(connection.Key);
// Elasticsearch.Net
builder.Register(ctx => {
var settings = new ConnectionConfiguration(ctx.ResolveNamed <IConnectionPool>(connection.Key));
if (_process.Mode != "init" && connection.RequestTimeout >= 0)
{
settings.RequestTimeout(new TimeSpan(0, 0, 0, connection.RequestTimeout * 1000));
}
if (connection.Timeout > 0)
{
settings.PingTimeout(new TimeSpan(0, 0, connection.Timeout));
}
return(new ElasticLowLevelClient(settings));
}).Named <IElasticLowLevelClient>(connection.Key);
// Process-Level Schema Reader
builder.Register <ISchemaReader>(ctx => new ElasticSchemaReader(ctx.ResolveNamed <IConnectionContext>(connection.Key), ctx.ResolveNamed <IElasticLowLevelClient>(connection.Key))).Named <ISchemaReader>(connection.Key);
// Entity Level Schema Readers
foreach (var entity in _process.Entities.Where(e => e.Connection == connection.Name))
{
builder.Register <ISchemaReader>(ctx => new ElasticSchemaReader(ctx.ResolveNamed <IConnectionContext>(entity.Key), ctx.ResolveNamed <IElasticLowLevelClient>(connection.Key))).Named <ISchemaReader>(entity.Key);
}
}
// Entity Input
foreach (var entity in _process.Entities.Where(e => _process.Connections.First(c => c.Name == e.Connection).Provider == "elasticsearch"))
{
builder.Register <IInputVersionDetector>(ctx => {
var input = ctx.ResolveNamed <InputContext>(entity.Key);
switch (input.Connection.Provider)
{
case "elasticsearch":
return(new ElasticInputVersionDetector(input, ctx.ResolveNamed <IElasticLowLevelClient>(input.Connection.Key)));
default:
return(new NullVersionDetector());
}
}).Named <IInputVersionDetector>(entity.Key);
// INPUT READER
builder.Register <IRead>(ctx => {
var input = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity));
switch (input.Connection.Provider)
{
case "elasticsearch":
return(new ElasticReader(input, input.InputFields, ctx.ResolveNamed <IElasticLowLevelClient>(input.Connection.Key), rowFactory, ReadFrom.Input));
default:
return(new NullReader(input, false));
}
}).Named <IRead>(entity.Key);
}
// Entity Output
if (_process.Output().Provider == "elasticsearch")
{
// PROCESS OUTPUT CONTROLLER
builder.Register <IOutputController>(ctx => new NullOutputController()).As <IOutputController>();
// PROCESS INITIALIZER
builder.Register <IInitializer>(ctx => {
var output = ctx.Resolve <OutputContext>();
return(new ElasticInitializer(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)));
}).As <IInitializer>();
foreach (var entity in _process.Entities)
{
// UPDATER
builder.Register <IUpdate>(ctx => {
var output = ctx.ResolveNamed <OutputContext>(entity.Key);
output.Warn($"{output.Connection.Provider} does not denormalize.");
return(new NullMasterUpdater());
}).Named <IUpdate>(entity.Key);
// OUTPUT
builder.Register <IOutputController>(ctx => {
var output = ctx.ResolveNamed <OutputContext>(entity.Key);
switch (output.Connection.Provider)
{
case "elasticsearch":
var initializer = _process.Mode == "init" ? (IAction) new ElasticEntityInitializer(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)) : new NullInitializer();
return(new ElasticOutputController(
output,
initializer,
ctx.ResolveNamed <IInputVersionDetector>(entity.Key),
new ElasticOutputVersionDetector(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)),
ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)
));
default:
return(new NullOutputController());
}
}).Named <IOutputController>(entity.Key);
// WRITER
builder.Register <IWrite>(ctx => {
var output = ctx.ResolveNamed <OutputContext>(entity.Key);
switch (output.Connection.Provider)
{
case "elasticsearch":
return(new ElasticWriter(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)));
default:
return(new NullWriter(output));
}
}).Named <IWrite>(entity.Key);
// DELETE HANDLER
if (entity.Delete)
{
builder.Register <IEntityDeleteHandler>(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
var inputContext = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", inputContext.RowCapacity));
IRead input = new NullReader(context);
var primaryKey = entity.GetPrimaryKey();
switch (inputContext.Connection.Provider)
{
case "elasticsearch":
input = new ElasticReader(
inputContext,
primaryKey,
ctx.ResolveNamed <IElasticLowLevelClient>(inputContext.Connection.Key),
rowFactory,
ReadFrom.Input
);
break;
}
IRead output = new NullReader(context);
IDelete deleter = new NullDeleter(context);
var outputConnection = _process.Output();
var outputContext = ctx.ResolveNamed <OutputContext>(entity.Key);
switch (outputConnection.Provider)
{
case "elasticsearch":
output = new ElasticReader(
outputContext,
primaryKey,
ctx.ResolveNamed <IElasticLowLevelClient>(inputContext.Connection.Key),
rowFactory,
ReadFrom.Output
);
deleter = new ElasticPartialUpdater(
outputContext,
new[] { context.Entity.TflDeleted() },
ctx.ResolveNamed <IElasticLowLevelClient>(inputContext.Connection.Key)
);
break;
}
var handler = new DefaultDeleteHandler(context, input, output, deleter);
// since the primary keys from the input may have been transformed into the output, you have to transform before comparing
// feels a lot like entity pipeline on just the primary keys... may look at consolidating
handler.Register(new DefaultTransform(context, entity.GetPrimaryKey().ToArray()));
handler.Register(TransformFactory.GetTransforms(ctx, _process, entity, primaryKey));
handler.Register(new StringTruncateTransfom(context, primaryKey));
return(new ParallelDeleteHandler(handler));
}).Named <IEntityDeleteHandler>(entity.Key);
}
}
}
}
public void Build()
{
//MAPS
foreach (var map in _process.Maps.Where(m => m.Connection != string.Empty && m.Query != string.Empty))
{
var connection = _process.Connections.First(c => c.Name == map.Connection);
if (connection != null && connection.Provider == "elasticsearch")
{
_builder.Register <IMapReader>(ctx => new DefaultMapReader()).Named <IMapReader>(map.Name);
}
}
//CONNECTIONS
foreach (var connection in _process.Connections.Where(c => c.Provider == "elasticsearch"))
{
if (connection.Servers.Any(s => s.Url != "None"))
{
var uris = new List <Uri>();
foreach (var server in connection.Servers.Where(s => s.Url != "None"))
{
server.Url = server.GetElasticUrl();
uris.Add(new Uri(server.Url));
}
// for now, just use static connection pool, there are 2 other types...
_builder.Register <IConnectionPool>(ctx => new StaticConnectionPool(uris)).Named <IConnectionPool>(connection.Key);
}
else
{
connection.Url = connection.GetElasticUrl();
_builder.Register <IConnectionPool>(ctx => new SingleNodeConnectionPool(new Uri(connection.Url))).Named <IConnectionPool>(connection.Key);
}
// Elasticsearch.Net
_builder.Register(ctx => {
var settings = new ConnectionConfiguration(ctx.ResolveNamed <IConnectionPool>(connection.Key));
if (!string.IsNullOrEmpty(connection.User))
{
settings.BasicAuthentication(connection.User, connection.Password);
}
if (_process.Mode != "init" && connection.RequestTimeout >= 0)
{
settings.RequestTimeout(new TimeSpan(0, 0, 0, connection.RequestTimeout * 1000));
}
if (connection.Timeout > 0)
{
settings.PingTimeout(new TimeSpan(0, 0, connection.Timeout));
}
return(new ElasticLowLevelClient(settings));
}).Named <IElasticLowLevelClient>(connection.Key);
// Process-Level Schema Reader
_builder.Register <ISchemaReader>(ctx => new ElasticSchemaReader(ctx.ResolveNamed <IConnectionContext>(connection.Key), ctx.ResolveNamed <IElasticLowLevelClient>(connection.Key))).Named <ISchemaReader>(connection.Key);
// Entity Level Schema Readers
foreach (var entity in _process.Entities.Where(e => e.Input == connection.Name))
{
_builder.Register <ISchemaReader>(ctx => new ElasticSchemaReader(ctx.ResolveNamed <IConnectionContext>(entity.Key), ctx.ResolveNamed <IElasticLowLevelClient>(connection.Key))).Named <ISchemaReader>(entity.Key);
}
}
// Entity Input
foreach (var entity in _process.Entities.Where(e => _process.Connections.First(c => c.Name == e.Input).Provider == "elasticsearch"))
{
_builder.Register <IInputProvider>(ctx => {
var input = ctx.ResolveNamed <InputContext>(entity.Key);
return(new ElasticInputProvider(input, ctx.ResolveNamed <IElasticLowLevelClient>(input.Connection.Key)));
}).Named <IInputProvider>(entity.Key);
// INPUT READER
_builder.Register <IRead>(ctx => {
var input = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", input.RowCapacity));
if (entity.Query == string.Empty)
{
return(new ElasticReader(input, input.InputFields, ctx.ResolveNamed <IElasticLowLevelClient>(input.Connection.Key), rowFactory, ReadFrom.Input));
}
return(new ElasticQueryReader(input, ctx.ResolveNamed <IElasticLowLevelClient>(input.Connection.Key), rowFactory));
}).Named <IRead>(entity.Key);
}
// Entity Output
if (_process.GetOutputConnection().Provider == "elasticsearch")
{
// PROCESS OUTPUT CONTROLLER
_builder.Register <IOutputController>(ctx => new NullOutputController()).As <IOutputController>();
// PROCESS INITIALIZER
_builder.Register <IInitializer>(ctx => {
var output = ctx.Resolve <OutputContext>();
return(new ElasticInitializer(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)));
}).As <IInitializer>();
foreach (var entity in _process.Entities)
{
// UPDATER
_builder.Register <IUpdate>(ctx => {
var output = ctx.ResolveNamed <OutputContext>(entity.Key);
output.Debug(() => $"{output.Connection.Provider} does not denormalize.");
return(new NullMasterUpdater());
}).Named <IUpdate>(entity.Key);
// OUTPUT
_builder.Register <IOutputController>(ctx => {
var output = ctx.ResolveNamed <OutputContext>(entity.Key);
switch (output.Connection.Provider)
{
case "elasticsearch":
var initializer = _process.Mode == "init" ? (IAction) new ElasticEntityInitializer(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)) : new NullInitializer();
return(new ElasticOutputController(
output,
initializer,
ctx.ResolveNamed <IInputProvider>(entity.Key),
new ElasticOutputProvider(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)),
ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)
));
default:
return(new NullOutputController());
}
}).Named <IOutputController>(entity.Key);
// WRITER
_builder.Register <IWrite>(ctx => {
var output = ctx.ResolveNamed <OutputContext>(entity.Key);
switch (output.Connection.Provider)
{
case "elasticsearch":
return(new ElasticWriter(output, ctx.ResolveNamed <IElasticLowLevelClient>(output.Connection.Key)));
default:
return(new NullWriter(output));
}
}).Named <IWrite>(entity.Key);
// DELETE HANDLER
if (entity.Delete)
{
_builder.Register <IEntityDeleteHandler>(ctx => {
var context = ctx.ResolveNamed <IContext>(entity.Key);
var inputContext = ctx.ResolveNamed <InputContext>(entity.Key);
var rowFactory = ctx.ResolveNamed <IRowFactory>(entity.Key, new NamedParameter("capacity", inputContext.RowCapacity));
IRead input = new NullReader(context);
var primaryKey = entity.GetPrimaryKey();
switch (inputContext.Connection.Provider)
{
case "elasticsearch":
input = new ElasticReader(
inputContext,
primaryKey,
ctx.ResolveNamed <IElasticLowLevelClient>(inputContext.Connection.Key),
rowFactory,
ReadFrom.Input
);
break;
}
IRead output = new NullReader(context);
IDelete deleter = new NullDeleter(context);
var outputConnection = _process.GetOutputConnection();
var outputContext = ctx.ResolveNamed <OutputContext>(entity.Key);
switch (outputConnection.Provider)
{
case "elasticsearch":
output = new ElasticReader(
outputContext,
primaryKey,
ctx.ResolveNamed <IElasticLowLevelClient>(inputContext.Connection.Key),
rowFactory,
ReadFrom.Output
);
deleter = new ElasticPartialUpdater(
outputContext,
new[] { context.Entity.TflDeleted() },
ctx.ResolveNamed <IElasticLowLevelClient>(inputContext.Connection.Key)
);
break;
}
var handler = new DefaultDeleteHandler(context, input, output, deleter);
// since the primary keys from the input may have been transformed into the output, you have to transform before comparing
// feels a lot like entity pipeline on just the primary keys... may look at consolidating
handler.Register(new DefaultTransform(context, entity.GetPrimaryKey().ToArray()));
handler.Register(TransformFactory.GetTransforms(ctx, context, primaryKey));
handler.Register(new StringTruncateTransfom(context, primaryKey));
return(handler);
}).Named <IEntityDeleteHandler>(entity.Key);
}
}
}
}