public bool InsertMany(List <T> insertObjList, out string message) { if (_elkClient != null) { var bulkIndexResponse = _elkClient.Bulk(b => b .Index(_index) .IndexMany(insertObjList)); if (bulkIndexResponse.IsValid && bulkIndexResponse.ApiCall.Success && bulkIndexResponse.DebugInformation.ToLower().Contains("valid nest response")) { message = "Success"; return(true); } else { message = bulkIndexResponse.DebugInformation; return(false); } } else { message = "Cert File does not exists"; return(false); } }
public virtual void PopulateData() { var descriptor = new BulkDescriptor(); AddElementsToIndex(descriptor); var response = _client.Bulk(descriptor); LogResponseAndSleep(response); }
private static List <Func <object> > DocumentCommands(ElasticClient elastic) { return(new List <Func <object> > { () => elastic.Bulk(new BulkRequest("test_index") { Operations = new List <IBulkOperation> { new BulkCreateOperation <Post>(new Post { Id = 1, Title = "BulkCreateOperation", }) } }), () => elastic.Create <Post>(new Post { Id = 2, Title = "Create", }), // () => elastic.CreateDocument<Post>(new Post // { // Id = 3, // Title = "CreateDocument", // }), // V6 Feature () => elastic.Count <Post>(), () => elastic.Search <Post>(s => s.MatchAll()), () => elastic.DeleteByQuery(new DeleteByQueryRequest("test_index") { Size = 0, }), }); }
static void Main(string[] args) { // load the sample data - a list of countries scraped from Reuters articles var lines = File.ReadAllLines(dataPath); var countries = from l in lines select new Country() { Name = l }; Console.WriteLine($"Found {countries.Count()} documents ready for indexing..."); // ask for the admin password, so I can safely store this code in github Console.Write("Please provide password for Elastic cluster: "); var password = Console.ReadLine(); // connect to elasticsearch using basic auth var credentials = new BasicAuthenticationCredentials("elastic", password); var cloudId = "Test:d2VzdGV1cm9wZS5henVyZS5lbGFzdGljLWNsb3VkLmNvbTo5MjQzJDI3MTE2MzJhMDBjZTRmNmZhNDE1MWU2ZmU2NDYxYTU3JDYwMGRhOTBhNDUwMzRmNmE4MDA5NDg2OWIxZDZhYmJi"; var settings = new ConnectionSettings(cloudId, credentials); var client = new ElasticClient(settings); // index the country objects using bulk indexing var response = client.Bulk(b => b.Index("countries").IndexMany(countries)); // report any errors if (response.Errors) { Console.WriteLine(response.ServerError.ToString()); } else { Console.WriteLine($"Indexed {countries.Count()} new documents!"); } }
private static void SeedIndex(ElasticClient client) { var faker = new Faker <PersonDocument>(); faker .RuleFor(r => r.Id, f => Guid.NewGuid().ToString()) .RuleFor(r => r.Name, f => $"{f.Person.FirstName} {f.Person.LastName}") .RuleFor(r => r.Age, f => f.Random.Number(16, 65)) .RuleFor(r => r.Country, f => FilterField.Create(f.Address.Country(), f.Address.CountryCode().ToLowerInvariant())) .RuleFor(r => r.Tags, f => f.Commerce.Categories(5).Select(FilterField.Create)) ; var list = faker.Generate(100).ToList(); list.Add(new PersonDocument { Id = Guid.NewGuid().ToString(), Name = "Phil Oyston", Age = 20, Country = FilterField.Create("United Kingdom", "uk"), Tags = new[] { FilterField.Create("Baby") } }); list.Add(new PersonDocument { Id = Guid.NewGuid().ToString(), Name = "John Doe", Age = 30, Country = FilterField.Create("United Kingdom", "uk"), Tags = new[] { FilterField.Create("Grocery") } }); list.Add(new PersonDocument { Id = Guid.NewGuid().ToString(), Name = "John Smith", Age = 40, Country = FilterField.Create("United Kingdom", "uk"), Tags = new[] { FilterField.Create("Baby"), FilterField.Create("Grocery") } }); client.Bulk(b => b.IndexMany(list).Refresh(Refresh.True)); }
public static void Bulk_Index_Iterate(string url) { var uri = new Uri(url); var config = new ConnectionSettings(uri); config.SetDefaultIndex("rolling-stone-500"); var client = new ElasticClient(config); List <Album> albums = new List <Album> { new Album(), new Album(), new Album(), }; var bulkDescriptor = new BulkDescriptor(); foreach (var album in albums) { bulkDescriptor.Index <Album>(bid => bid .Id(album.Rank) .Document(album) ); } var result = client.Bulk(bulkDescriptor); }
public static void Do_it(ElasticClient client) { var people = new[] { new Person { Id = 1, FirstName = "Martijn", LastName = "Laarman" }, new Person { Id = 2, FirstName = "Stuart", LastName = "Cam" }, new Person { Id = 3, FirstName = "Russ", LastName = "Cam" } }; var bulkIndexResponse = client.Bulk(b => b .Index("people") .IndexMany(people)); }
private static List <Func <object> > DocumentCommands(ElasticClient elastic) { return(new List <Func <object> > { () => elastic.Bulk(new BulkRequest("test_index") { Operations = new List <IBulkOperation> { new BulkCreateOperation <Post>(new Post { Id = 1, Title = "BulkCreateOperation", }) } }), () => elastic.Create <Post>(new CreateRequest <Post>(new Post { Id = 2, Title = "CreateRequest", }, "test_index")), () => elastic.CreateDocument <Post>(new Post { Id = 3, Title = "CreateDocument", }), () => elastic.Count <Post>(), () => elastic.DeleteByQuery(new DeleteByQueryRequest("test_index") { Size = 0, }), }); }
public void WhenPostExceedsHttpLimit_DoNotRetry_UsingConnectionPooling() { var pool = new StaticConnectionPool(new [] { new Uri("http://localhost:9200"), new Uri("http://127.0.0.1:9200"), }); var settings = new ConnectionSettings(pool); var client = new ElasticClient(settings); var index = ElasticsearchConfiguration.NewUniqueIndexName(); var projects = NestTestData.Data; var people = NestTestData.People; var boolTerms = NestTestData.BoolTerms; var bulk = client.Bulk(b => b .FixedPath(index) .IndexMany(projects) .IndexMany(people) .IndexMany(boolTerms) ); bulk.IsValid.Should().BeFalse(); bulk.ConnectionStatus.NumberOfRetries.Should().Be(0); }
public static void UpdateElasticSearch() { var settings = new ConnectionSettings(new Uri("https://search-search-rvezy-sdxhps7oh6vgvoy7f3ctpxnh7q.us-east-2.es.amazonaws.com")).DefaultIndex("rv-local"); var client = new ElasticClient(settings); var descriptor = new BulkDescriptor(); for (int i = 0; i < 2; i++) { var id = "a306e70e-1ab1-4dde-9c58-31e677762af6"; var score = 8.8; if (i == 1) { id = "8fbfb724-8ebb-4bc0-82b0-5a1cf52fd2cc"; score = 10; } descriptor.Update <RV>(u => u .Id(id) .Doc(new RV { Id = new Guid(id), Score = score })); } var result = client.Bulk(descriptor); }
private void Push(ElasticClient client, IReadOnlyCollection <string> buffer) { Logger.Verbose("A buffer of {count} records is about to be sent for bulk indexing", buffer.Count); var descriptor = new BulkDescriptor(); foreach (var line in buffer) { var instance = JObject.Parse(line); var identifier = instance.Property("_id"); var id = identifier.Value.ToString(); identifier.Remove(); descriptor .Index <JObject>(i => i .Index("data") .Type("geo") .Id(id) .Document(instance)); } var response = client.Bulk(descriptor); if (response.Errors) { Logger.Debug("Bulk insert failed: {errors}", response.ItemsWithErrors); } }
public void NodesBulkInsert(string nodesTableName, List <Node> list) { var nodesBulkDescriptor = new BulkDescriptor(nodesTableName); Watch.Restart(); list.ForEach( node => nodesBulkDescriptor.Index <Dictionary <string, object> >( i => i.Document(new Dictionary <string, object> { ["name"] = node.Name }).Id(node.Id) ) ); _client.Bulk(nodesBulkDescriptor).Validate(); Console.WriteLine("Bulk node insert in " + Watch.ElapsedMilliseconds + " ms. count: " + list.Count); }
static void ImportSales() { var rnd = new Random(); var sales = new List <Sale>(); int count = 0; for (int i = 0; i < 2000000; i++) { var product = _products.FirstOrDefault(p => p.Id == rnd.Next(1, _products.Count())) ?? _products.First(); sales.Add(new Sale { Id = i, Date = DateTime.Now.AddYears(rnd.Next(-3, 1)).AddMonths(rnd.Next(-12, 1)).AddDays(rnd.Next(-30, 1)).AddHours(rnd.Next(-24, 1)), Distribution = i % 2 == 0? "website":"mobile", IsReturned = i % 100 == 0, ProductFamilies = product.ProductFamilies, ProductName = product.Name, ProductSku = product.Sku, SerialNumber = rnd.Next(99999999, 999999999).ToString(), RetailerId = rnd.Next(0, 200), SellerId = new Guid("00000000-0000-0000-0000-0000000" + rnd.Next(10000, 99999).ToString()) }); count++; if (count == 10000) { var resp = _elasticClient.Bulk(b => b .IndexMany(sales, (d, sale) => d.Document(sale).Index("mysaleindex"))); count = 0; sales = new List <Sale>(); } } }
static void Main(string[] args) { var totalThreads = 4; var bulkRequests = 500; var node = new Uri("http://myserver:9200"); var token = cancelToken; for (int i = 0; i < totalThreads; i++) { Task.Run(() => { while (!token.IsCancellationRequested) { var settings = new ConnectionSettings(node); var client = new ElasticClient(settings); var bulkRequest = new BulkRequest() { Operations = GetOperations(bulkRequests) }; var result = client.Bulk(bulkRequest); Console.WriteLine("Thread {0} publushed {1} requests with a status of {2}", i, result.IsValid, bulkRequests); } } , cancelToken); } }
static int Populate(ElasticClient client) { Text WithToken(Text text, int idx) { return(new Text { Id = text.Id, Index = text.Index, Length = text.Length, WordCount = text.WordCount, Content = text.Content, Token = idx % 2 == 0 ? LeftToken : RightToken, }); } using (var connection = new SqlConnection(DbConnectionString)) { var texts = connection.Query <Text>(TextQuery).Select(WithToken); var result = client.Bulk(bulk => bulk .Index(IndexName) .CreateMany(texts, (op, text) => op.Id(text.Id))); return(result.Items.Count); } }
/// <summary> /// 批量局部更新 /// 注意:此方法将使用默认索引 /// </summary> /// <param name="models">数据集合</param> /// <param name="partial">局部数据集合</param> /// <param name="isThrow">抛出异常</param> /// <returns></returns> public bool UpdateBulk <T>( List <T> models, List <object> partial, bool isThrow = false) where T : class { if (models == null || partial == null) { return(false); } if (models.Count != partial.Count) { throw new Exception("集合数量不一致"); } var Bulk = new BulkDescriptor().Index(IndiceName); for (int i = 0; i < models.Count; i++) { Bulk.Update <T, object>(u => u.IdFrom(models[i]).Doc(partial[i])); } var response = ElasticClient.Bulk(Bulk); if (!response.IsValid && isThrow) { throw new ElasticsearchException(response); } return(response.IsValid); }
/// <summary> /// 推送ES数据列表 /// </summary> /// <typeparam name="T">数据模型</typeparam> /// <param name="index">索引名称 [db]</param> /// <param name="type">索引类型 [table]</param> /// <param name="primarykeyName">推送的主键名称</param> /// <param name="data">推送数据</param> /// <returns></returns> public static bool Pulk <T>(string index, string type, string primarykeyName, List <T> data) where T : class { var bulkRequest = new BulkRequest(index, type) { Operations = new List <IBulkOperation>() }; var indexs = data.Select(p => new BulkIndexOperation <T>(p) { Id = p.GetType().GetProperty(primarykeyName).GetValue(p, null).ToString() }) .Cast <IBulkOperation>() .ToList(); bulkRequest.Operations = indexs; var response = client.Bulk(bulkRequest); if (response.OriginalException != null) { throw new ArgumentException(response.OriginalException.Message, response.OriginalException); } if (DebuggerMode && response.ApiCall != null) { var url = response.ApiCall.Uri.AbsoluteUri; var query = System.Text.Encoding.UTF8.GetString(response.ApiCall.RequestBodyInBytes); var responseText = System.Text.Encoding.UTF8.GetString(response.ApiCall.ResponseBodyInBytes); Debug.WriteLine(string.Format("url:{0}\t\nquery:{1}\t\n response:{2}", url, query, responseText)); } return(response.IsValid); }
public static void Insert(List <Novel> list) { ElasticClient elastic = GetElasticClient(); var Descriptor = new BulkDescriptor(); Descriptor.CreateMany(list); var result = elastic.Bulk(Descriptor); }
/// <summary> /// 保存最新的文件(和普通保存的区别是本方法不会删除文件,入库失败后会回写当前解析的内容到文件中) /// </summary> /// <param name="file">文件</param> private void SaveLatestLog(FileInfo file) { string content = string.Empty; try { List <SysLog> sysLogs; if (file.Name.StartsWith("Error")) { var exlog = GetLogModelAndClearFile <ExceptionLog>(file, out content); if (exlog == null || !exlog.Any()) { return; } sysLogs = BuildSysLog(exlog); } else { var log = GetLogModelAndClearFile <LogBase>(file, out content); if (log == null || !log.Any()) { return; } sysLogs = BuildSysLog(log); } BulkDescriptor descriptor = new BulkDescriptor(); foreach (SysLog sysLog in sysLogs) { descriptor.Index <SysLog>(op => op.Document(sysLog).Index("systemlog")); } var insertRes = _client.Bulk(descriptor); if (!insertRes.Errors) { file.Delete(); } } catch (Exception e) { WriteFileContent(file.FullName, content); LogService.WriteLog(e, "日志入库异常,信息:" + file.FullName); } }
public void UpdateViewCount(IDictionary <int, long> visits) { var result = _client.Bulk(bd => visits.Aggregate(bd, (b, kvp) => b.Update <BlogIndexed, object>(bud => bud.Id(kvp.Key).Doc(new { BlogVisit = kvp.Value })), i => i.Refresh(Elasticsearch.Net.Refresh.True))); if (!result.IsValid) { _logger.LogError(result.DebugInformation); } }
public void BulkInsert <T>(IEnumerable <T> input) where T : class, new() { _client.Bulk(new BulkRequest() { Operations = input.Select(o => (new BulkIndexOperation <T>(o)) as IBulkOperation ).ToList() }); }
public IActionResult SeedData() { if (_client.IndexExists("my_blog").Exists) { return(Ok("The Index 'my_blog' already exists.")); } //Mapping required? - Yes for Attributes on Models to work var indexDescriptor = new CreateIndexDescriptor("my_blog") .Mappings(ms => ms .Map <Post>(m => m.AutoMap() )); _client.CreateIndex("my_blog", i => indexDescriptor); //IndexCustomBlogs(); //Adds test data from a json text file var convertor = new JsonDataConvertor(); var listWithPostsRoots = convertor.JsonToPostList(); foreach (var postRoot in listWithPostsRoots) { var post = new Post() { UserId = postRoot.userId, Title = postRoot.title, Tags = postRoot.tags, Category = postRoot.category, SubCategory = postRoot.subCategory, PostDate = Convert.ToDateTime(postRoot.postDate), PostText = postRoot.postText }; //var result = _client.Index(post, idx => idx.Index("my_blog")); var result = _client.Bulk(b => b .Index <Post>(i => i.Document(post)) ); if (!result.IsValid) { return(Ok(result.DebugInformation)); } } if (_client.IndexExists("my_blog").Exists) { return(Ok($"Blog Posts has been sucessfully created.")); } else { return(NotFound($"Something went wrong.")); } }
public static void Indexing() { using (LocationContext dbContext = new LocationContext()) { var customerLocationList = dbContext.CustomerLocations.Where(s => !s.IsDeleted) .Include(s => s.Customer) .Select(s => new CustomerModel { CustomerId = s.CustomerId, CustomerName = s.Customer.Name, LocationId = s.Id, LocationName = s.Name, Location = new GeoLocation(Convert.ToDouble(s.Latitude), Convert.ToDouble(s.Longitude)) }).ToList(); var defaultIndex = "customerlocation"; var client = new ElasticClient(); if (client.IndexExists(defaultIndex).Exists) { client.DeleteIndex(defaultIndex); } if (!elasticClient.IndexExists("location_alias").Exists) { client.CreateIndex(defaultIndex, c => c .Mappings(m => m .Map <CustomerModel>(mm => mm .AutoMap() ) ).Aliases(a => a.Alias("location_alias")) ); } // Insert Data Classic // for (int i = 0; i < customerLocationList.Count; i++) // { // var item = customerLocationList[i]; // elasticClient.Index<CustomerModel>(item, idx => idx.Index("customerlocation").Id(item.LocationId)); // } // Bulk Insert var bulkIndexer = new BulkDescriptor(); foreach (var document in customerLocationList) { bulkIndexer.Index <CustomerModel>(i => i .Document(document) .Id(document.LocationId) .Index("customerlocation")); } elasticClient.Bulk(bulkIndexer); } }
public string InsertBulk(List <Product> products, string index) { var descriptor = new BulkDescriptor(); descriptor.Index(new IndexName() { Name = index }); foreach (var product in products) { //For each Product in Product-list, index it to the given index descriptor.Index <Product>(i => i.Document(product)); } string response = client.Bulk(descriptor).ToString(); return(response); }
public void bulkIndexCreate(string indexName, Object modelList) { var descriptor = new BulkDescriptor(); descriptor.Index <ElasticDataModel>(op => op .Document(modelList as ElasticDataModel) ); var result = esClient.Bulk(descriptor); }
public static void Setup() { var client = new ElasticClient(ElasticsearchConfiguration.Settings(hostOverride: new Uri("http://localhost:9200"))); //uncomment the next line if you want to see the setup in fiddler too //var client = ElasticsearchConfiguration.Client; var projects = NestTestData.Data; var people = NestTestData.People; var boolTerms = NestTestData.BoolTerms; try { var createIndexResult = client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c .NumberOfReplicas(0) .NumberOfShards(1) .AddMapping<ElasticsearchProject>(m => m .MapFromAttributes() .Properties(p => p .String(s => s.Name(ep => ep.Content).TermVector(TermVectorOption.WithPositionsOffsetsPayloads)))) .AddMapping<Person>(m => m.MapFromAttributes()) .AddMapping<BoolTerm>(m => m.Properties(pp => pp .String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.NotAnalyzed)) .String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.NotAnalyzed)) )) ); var createAntotherIndexResult = client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c .NumberOfReplicas(0) .NumberOfShards(1) .AddMapping<ElasticsearchProject>(m => m .MapFromAttributes() .Properties(p => p .String(s => s.Name(ep => ep.Content).TermVector(TermVectorOption.WithPositionsOffsetsPayloads)))) .AddMapping<Person>(m => m.MapFromAttributes()) .AddMapping<BoolTerm>(m => m.Properties(pp => pp .String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.NotAnalyzed)) .String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.NotAnalyzed)) )) ); var bulkResponse = client.Bulk(b => b .IndexMany(projects) .IndexMany(people) .IndexMany(boolTerms) .Refresh() ); } catch (Exception e) { throw; } }
public void InsertTransactions(IEnumerable <Transaction> transactions, ElasticClient client) { var descriptor = new BulkDescriptor(); foreach (var transaction in transactions) { descriptor.Create <Transaction>(op => op.Document(transaction)); } var result = client.Bulk(descriptor); }
public void InsertCards(IEnumerable <Card> cards, ElasticClient client) { var descriptor = new BulkDescriptor(); foreach (var card in cards) { descriptor.Create <Card>(op => op.Document(card)); } var result = client.Bulk(descriptor); }
public IBulkResponse BulkInsertCandidates(IEnumerable <VacancyElasticModel> candidates) { var bulk = new BulkDescriptor(); candidates.ForEach(cand => bulk.Index <VacancyElasticModel>(i => i .Index(IndexName) .Id(cand.Id) .Document(cand) )); return(_client.Bulk(bulk)); }
public static void InsertBulkDocument() { var descriptor = new BulkDescriptor(); foreach (var employeeObj in PopulateEmployees()) { Employee obj = employeeObj; descriptor.Index <Employee>(op => op.Document(obj)); } var bulkresult = EsClient.Bulk(descriptor); }
public static void Setup() { var client = new ElasticClient(ElasticsearchConfiguration.Settings(hostOverride: new Uri("http://localhost:9200"))); //uncomment the next line if you want to see the setup in fiddler too //var client = ElasticsearchConfiguration.Client; var projects = NestTestData.Data; var people = NestTestData.People; var boolTerms = NestTestData.BoolTerms; try { var createIndexResult = client.CreateIndex(ElasticsearchConfiguration.DefaultIndex, c => c .NumberOfReplicas(0) .NumberOfShards(1) .AddMapping <ElasticsearchProject>(m => m .MapFromAttributes() .Properties(p => p .String(s => s.Name(ep => ep.Content).TermVector(TermVectorOption.WithPositionsOffsetsPayloads)))) .AddMapping <Person>(m => m.MapFromAttributes()) .AddMapping <BoolTerm>(m => m.Properties(pp => pp .String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.NotAnalyzed)) .String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.NotAnalyzed)) )) ); var createAntotherIndexResult = client.CreateIndex(ElasticsearchConfiguration.DefaultIndex + "_clone", c => c .NumberOfReplicas(0) .NumberOfShards(1) .AddMapping <ElasticsearchProject>(m => m .MapFromAttributes() .Properties(p => p .String(s => s.Name(ep => ep.Content).TermVector(TermVectorOption.WithPositionsOffsetsPayloads)))) .AddMapping <Person>(m => m.MapFromAttributes()) .AddMapping <BoolTerm>(m => m.Properties(pp => pp .String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.NotAnalyzed)) .String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.NotAnalyzed)) )) ); var bulkResponse = client.Bulk(b => b .IndexMany(projects) .IndexMany(people) .IndexMany(boolTerms) .Refresh() ); } catch (Exception e) { throw; } }