private async Task <bool> PushBulk(IEnumerable <ElasticSDResult> results)
{
if (elasticClient == null)
{
return(false);
}
var descriptor = new BulkDescriptor();
descriptor.CreateMany <ElasticSDResult>(results);
Console.WriteLine($"Inserting {results.Count()} superdump results into ES...");
var sw = new Stopwatch();
sw.Start();
var result = await elasticClient.BulkAsync(descriptor);
sw.Stop();
if (result.IsValid)
{
Console.WriteLine($"Finished inserting in {sw.Elapsed})");
return(true);
}
// something failed
Console.WriteLine($"PushBulk failed for {result.ItemsWithErrors?.Count()} items. servererror: {result.ServerError?.ToString()}, error on first item: {result.ItemsWithErrors?.FirstOrDefault()?.Error}");
return(false);
}
public static void Insert(List <Novel> list)
{
ElasticClient elastic = GetElasticClient();
var Descriptor = new BulkDescriptor();
Descriptor.CreateMany(list);
var result = elastic.Bulk(Descriptor);
}
public Boolean CreateBulkDocument(String index, String type, List <Dictionary <String, Object> > obj)
{
var descriptor = new BulkDescriptor();
descriptor.CreateMany <Dictionary <String, Object> >(obj, (bd, q) => bd.Id(q["id"].ToString()).Index(index).Type(type));
var bulkResponse = _elasticClient.Bulk(descriptor);
var response = _elasticClient.BulkAll(obj, idx => idx
.Index(index)
.Type(type)
.Refresh(Refresh.True)
);
if (bulkResponse.ServerError != null && bulkResponse.ServerError.ToString().Length > 0)
{
Console.WriteLine("Error : {0}", bulkResponse.ServerError.ToString());
Console.WriteLine("Error Content : {0}", Newtonsoft.Json.JsonConvert.SerializeObject(obj));
}
return(bulkResponse.IsValid);
}
public async Task <IEnumerable <IndexResult> > IndexMany <T>(IEnumerable <SearchableItem <T> > items)
{
var retval = new List <IndexResult>();
foreach (var itemBatch in items.Batch(_configuration.BatchSize))
{
var request = new BulkDescriptor();
request.CreateMany(itemBatch).Index(_configuration.Index);
var result = await _client.BulkAsync(request);
if (result.ApiCall.HttpStatusCode.HasValue && result.ApiCall.HttpStatusCode.Value >= 300)
{
Log.Error(result.OriginalException, "Unable to bulk insert to elastic");
return(new IndexResult[0]);
}
foreach (var item in result.Items)
{
IndexResult indexResult = CreateErrorResult(item);
LogError(indexResult);
retval.Add(indexResult);
}
}
return(retval);
}
private static async Task SetupInternal()
{
var config = SvzConfig.GetConfig();
using (var conn = new NpgsqlConnection(config.Postgres.ConnectionString))
{
var dbExists =
(await conn.QueryAsync <int>(
$"SELECT 1 from pg_database WHERE datname = '{config.Postgres.Database}'")).ToList();
if (dbExists.Any())
{
Ms("Dropping existing Svz database... ");
await conn.ExecuteScalarAsync($"DROP DATABASE \"{config.Postgres.Database}\"");
Ok();
}
Ms("Creating new Svz database... ");
await conn.ExecuteScalarAsync($"CREATE DATABASE \"{config.Postgres.Database}\"");
Ok();
}
Ms("Running migrations... ");
var optBuild = new DbContextOptionsBuilder <BookDbContext>();
optBuild.UseNpgsql($"{config.Postgres.ConnectionString};Database={config.Postgres.Database};");
using (var ctx = new BookDbContext(optBuild.Options))
{
await ctx.Database.MigrateAsync();
Ok();
Ms("Import books from json... ");
await ImportBooks("./books.json", ctx);
Ok();
}
var esConnection = new ConnectionSettings(new Uri(config.ElasticSearch.Url));
var esClient = new ElasticClient(esConnection);
var idxResp = await esClient.GetIndexAsync(config.ElasticSearch.Index);
foreach (var item in idxResp.Indices)
{
Ms($"Dropping ES Index: {item.Key}... ");
var resp = await esClient.DeleteIndexAsync(item.Key);
if (!resp.IsValid)
{
throw new Exception("Error deleting ES index: " + resp.ServerError.Error);
}
Ok();
}
Ms("Creating ES index... ");
var respCreate = await esClient.CreateIndexAsync(config.ElasticSearch.Index);
if (!respCreate.IsValid)
{
throw new Exception("Error creating ES index: " + respCreate.ServerError.Error);
}
Ok();
Ms("Indexing data... ");
Mapper.Initialize(cfg => { cfg.CreateMap <Book, BookSearchView>(); });
var bsearch = new List <BookSearchView>();
using (var ctx = new BookDbContext(optBuild.Options))
{
var blist = ctx.Books.ToList();
bsearch = Mapper.Map <List <Book>, List <BookSearchView> >(blist);
}
var bulkDesc = new BulkDescriptor();
bulkDesc.CreateMany(bsearch, (x, y) => x.Id(y.BookId)).Index(config.ElasticSearch.Index);
var respBulk = await esClient.BulkAsync(bulkDesc);
if (!respBulk.IsValid)
{
throw new Exception("Error indexing ES data: " + respBulk.ServerError);
}
Ok();
}
