private async Task <bool> PushBulk(IEnumerable <ElasticSDResult> results) { if (elasticClient == null) { return(false); } var descriptor = new BulkDescriptor(); descriptor.CreateMany <ElasticSDResult>(results); Console.WriteLine($"Inserting {results.Count()} superdump results into ES..."); var sw = new Stopwatch(); sw.Start(); var result = await elasticClient.BulkAsync(descriptor); sw.Stop(); if (result.IsValid) { Console.WriteLine($"Finished inserting in {sw.Elapsed})"); return(true); } // something failed Console.WriteLine($"PushBulk failed for {result.ItemsWithErrors?.Count()} items. servererror: {result.ServerError?.ToString()}, error on first item: {result.ItemsWithErrors?.FirstOrDefault()?.Error}"); return(false); }
public static void Insert(List <Novel> list) { ElasticClient elastic = GetElasticClient(); var Descriptor = new BulkDescriptor(); Descriptor.CreateMany(list); var result = elastic.Bulk(Descriptor); }
public Boolean CreateBulkDocument(String index, String type, List <Dictionary <String, Object> > obj) { var descriptor = new BulkDescriptor(); descriptor.CreateMany <Dictionary <String, Object> >(obj, (bd, q) => bd.Id(q["id"].ToString()).Index(index).Type(type)); var bulkResponse = _elasticClient.Bulk(descriptor); var response = _elasticClient.BulkAll(obj, idx => idx .Index(index) .Type(type) .Refresh(Refresh.True) ); if (bulkResponse.ServerError != null && bulkResponse.ServerError.ToString().Length > 0) { Console.WriteLine("Error : {0}", bulkResponse.ServerError.ToString()); Console.WriteLine("Error Content : {0}", Newtonsoft.Json.JsonConvert.SerializeObject(obj)); } return(bulkResponse.IsValid); }
public async Task <IEnumerable <IndexResult> > IndexMany <T>(IEnumerable <SearchableItem <T> > items) { var retval = new List <IndexResult>(); foreach (var itemBatch in items.Batch(_configuration.BatchSize)) { var request = new BulkDescriptor(); request.CreateMany(itemBatch).Index(_configuration.Index); var result = await _client.BulkAsync(request); if (result.ApiCall.HttpStatusCode.HasValue && result.ApiCall.HttpStatusCode.Value >= 300) { Log.Error(result.OriginalException, "Unable to bulk insert to elastic"); return(new IndexResult[0]); } foreach (var item in result.Items) { IndexResult indexResult = CreateErrorResult(item); LogError(indexResult); retval.Add(indexResult); } } return(retval); }
private static async Task SetupInternal() { var config = SvzConfig.GetConfig(); using (var conn = new NpgsqlConnection(config.Postgres.ConnectionString)) { var dbExists = (await conn.QueryAsync <int>( $"SELECT 1 from pg_database WHERE datname = '{config.Postgres.Database}'")).ToList(); if (dbExists.Any()) { Ms("Dropping existing Svz database... "); await conn.ExecuteScalarAsync($"DROP DATABASE \"{config.Postgres.Database}\""); Ok(); } Ms("Creating new Svz database... "); await conn.ExecuteScalarAsync($"CREATE DATABASE \"{config.Postgres.Database}\""); Ok(); } Ms("Running migrations... "); var optBuild = new DbContextOptionsBuilder <BookDbContext>(); optBuild.UseNpgsql($"{config.Postgres.ConnectionString};Database={config.Postgres.Database};"); using (var ctx = new BookDbContext(optBuild.Options)) { await ctx.Database.MigrateAsync(); Ok(); Ms("Import books from json... "); await ImportBooks("./books.json", ctx); Ok(); } var esConnection = new ConnectionSettings(new Uri(config.ElasticSearch.Url)); var esClient = new ElasticClient(esConnection); var idxResp = await esClient.GetIndexAsync(config.ElasticSearch.Index); foreach (var item in idxResp.Indices) { Ms($"Dropping ES Index: {item.Key}... "); var resp = await esClient.DeleteIndexAsync(item.Key); if (!resp.IsValid) { throw new Exception("Error deleting ES index: " + resp.ServerError.Error); } Ok(); } Ms("Creating ES index... "); var respCreate = await esClient.CreateIndexAsync(config.ElasticSearch.Index); if (!respCreate.IsValid) { throw new Exception("Error creating ES index: " + respCreate.ServerError.Error); } Ok(); Ms("Indexing data... "); Mapper.Initialize(cfg => { cfg.CreateMap <Book, BookSearchView>(); }); var bsearch = new List <BookSearchView>(); using (var ctx = new BookDbContext(optBuild.Options)) { var blist = ctx.Books.ToList(); bsearch = Mapper.Map <List <Book>, List <BookSearchView> >(blist); } var bulkDesc = new BulkDescriptor(); bulkDesc.CreateMany(bsearch, (x, y) => x.Id(y.BookId)).Index(config.ElasticSearch.Index); var respBulk = await esClient.BulkAsync(bulkDesc); if (!respBulk.IsValid) { throw new Exception("Error indexing ES data: " + respBulk.ServerError); } Ok(); }