public static void IndexDemoData(IElasticClient client, string index = null) { index = index ?? ElasticsearchConfiguration.DefaultIndex; var projects = NestTestData.Data; var people = NestTestData.People; var boolTerms = NestTestData.BoolTerms; var parents = NestTestData.Parents; var children = NestTestData.Children; var bulkResponse = client.Bulk(b => { b.FixedPath(index); b.IndexMany(projects); b.IndexMany(people); b.IndexMany(boolTerms); var rand = new Random(); foreach (var parent in parents) b.Index<Parent>(i => i.Document(parent)); foreach (var child in children) b.Index<Child>(i => i .Document(child) .Parent(parents[rand.Next(parents.Count)].Id) ); b.Refresh(); return b; }); }
public static void IndexDemoData(IElasticClient client, string index = null) { index = index ?? ElasticsearchConfiguration.DefaultIndex; var projects = NestTestData.Data; var people = NestTestData.People; var boolTerms = NestTestData.BoolTerms; var bulkResponse = client.Bulk(b => b .FixedPath(index) .IndexMany(projects) .IndexMany(people) .IndexMany(boolTerms) .Refresh() ); }
public void ShouldAttemptToSaveIfMessagesReceived() { IElasticClient elasticClient = A.Fake <IElasticClient>(); TestScheduler testScheduler = new TestScheduler(); IMonitorFactory <TestMessage> factory = new ElasticSearchMonitorFactory <TestMessage>("indexName", new List <Type>(), "instanceName", TimeSpan.FromSeconds(1), testScheduler, elasticClient); IMonitor <TestMessage> monitor = factory.Create("SomeName"); monitor.MessageReceived(new TestMessage(), TimeSpan.FromMilliseconds(1)); testScheduler.AdvanceBy(TimeSpan.FromMinutes(1).Ticks); A.CallTo(() => elasticClient.Bulk(A <IBulkRequest> ._)).MustHaveHappened(1, Times.Exactly); }
public BulkResponse BulkInsert(IEnumerable<TEntity> entities) { var request = new BulkDescriptor(); foreach (var entity in entities) { request .Index<TEntity>(op => op .Id(Guid.NewGuid().ToString()) .Index(IndexName) .Document(entity)); } return Client.Bulk(request); }
public void ShouldNotAttemptToSaveIfNoMessages() { IElasticClient elasticClient = A.Fake <IElasticClient>(); TestScheduler testScheduler = new TestScheduler(); IMonitorFactory <TestMessage> factory = new ElasticSearchMonitorFactory <TestMessage>("indexName", new List <Type>(), "instanceName", TimeSpan.FromSeconds(1), testScheduler, elasticClient); IMonitor <TestMessage> monitor = factory.Create("SomeName"); Assert.NotNull(monitor); testScheduler.AdvanceBy(TimeSpan.FromMinutes(1).Ticks); A.CallTo(() => elasticClient.Bulk(A <IBulkRequest> ._)).MustNotHaveHappened(); }
public void Add() { BulkDescriptor bulkDescriptor = new BulkDescriptor(); bulkDescriptor.UpdateMany <IndexItem>(GetSampleData(), (b, u) => b.Index(_indexName) .Doc(u) .DocAsUpsert()); BulkResponse insert = _elasticClient.Bulk(bulkDescriptor); if (insert.IsValid is false) { throw new Exception(insert.OriginalException.ToString()); } }
/// <summary> /// Add documents to index using Bulk.<br/> /// Throws exception if inserting failed. /// </summary> /// <param name="items">An enumerable of IndexItems. </param> public void AddToIndex(IEnumerable <IIndexItem> items) { var bulkDescriptor = new BulkDescriptor(); foreach (var item in items) { bulkDescriptor.Index <IIndexItem>(x => x. Index(this.IndexName). Document(item) ); } var response = elasticClient.Bulk(bulkDescriptor); var validator = new ElasticResponseValidator(); validator.Validate(response); }
/** * ==== Multiple documents with bulk * * If you require finer grained control over indexing many documents you can use the `Bulk` and `BulkAsync` methods and use the descriptors to * customise the bulk calls. * * As with the `IndexMany` methods above, documents are sent to the `_bulk` endpoint in a single HTTP request. * This does mean that consideration will need to be given to the overall size of the HTTP request. For indexing large numbers * of documents it may be sensible to perform multiple separate `Bulk` calls. */ public async Task BulkIndexDocuments() { //hide var people = new [] { new Person { Id = 1, FirstName = "Martijn", LastName = "Laarman" } }; var bulkIndexResponse = client.Bulk(b => b .Index("people") .IndexMany(people)); //<1> synchronous method that returns an IBulkResponse, the same as IndexMany and can be inspected in the same way for errors // Alternatively, documents can be indexed asynchronously similar to IndexManyAsync var asyncBulkIndexResponse = await client.BulkAsync(b => b .Index("people") .IndexMany(people)); //<2> asynchronous method that returns a Task<IBulkResponse> that can be awaited }
public void InsertFilesInFolder(string folderPath) { var descriptor = new BulkDescriptor(); foreach (var doc in TextFileReader.ReadAllFilesInDirectory(folderPath)) { descriptor .Index <Document>(d => d .Document(doc) ); } _client .Bulk(descriptor) .Validate(); }
public bool IngestCargos(IEnumerable <Cargo> cargoList, out IEnumerable <Cargo> failures) { failures = new List <Cargo>(); IBulkResponse response = client.Bulk(b => b.IndexMany(cargoList, (op, c) => op.Document(c))); if (response.Errors) { failures = response.Items .Select((Item, Index) => new { Item, Index }) .Where(e => !e.Item.IsValid) .Select(e => cargoList.ElementAt(e.Index)); } return(!response.Errors); }
protected override void OnBeforeCall(IElasticClient client) { var create = client.Indices.Create(CallIsolatedValue, c => c .Aliases(a => a .Alias(CallIsolatedValue + "-alias") ) ); create.ShouldBeValid(); var someDocs = client.Bulk(b => b .Index(CallIsolatedValue) .Refresh(Refresh.True) .IndexMany(Project.Generator.Generate(1200)) ); someDocs.ShouldBeValid(); }
public void ShouldAttemptToSaveMultipleCountersIfMessagesReceived() { IElasticClient elasticClient = A.Fake <IElasticClient>(); TestScheduler testScheduler = new TestScheduler(); var types = new List <Type> { typeof(TestMessage) }; IMonitorFactory <TestMessage> factory = new ElasticSearchMonitorFactory <TestMessage>("indexName", types, "instanceName", TimeSpan.FromSeconds(1), testScheduler, elasticClient); IMonitor <TestMessage> monitor = factory.Create("SomeName"); monitor.MessageReceived(new TestMessage(), TimeSpan.FromMilliseconds(1)); testScheduler.AdvanceBy(TimeSpan.FromMinutes(1).Ticks); A.CallTo(() => elasticClient.Bulk(A <IBulkRequest> .That.Matches(request => request.Operations.Count == 1 + types.Count))).MustHaveHappened(1, Times.Exactly); }
public void Sync() { var bulkResponse = _client.Bulk(b => b .IndexMany(Developer.Generator.Generate(1000), (bd, d) => bd .Index(IndexName) .Document(d) )); if (!bulkResponse.IsValid) { if (bulkResponse.Errors) { foreach (var error in bulkResponse.ItemsWithErrors) { Console.WriteLine($"error with id {error.Id}. message: {error.Error.Reason}"); } } } }
internal static void SeedElasticsearch(this IApplicationBuilder app) { if (app == null) { throw new ArgumentNullException(nameof(app)); } IElasticClient client = app.ApplicationServices.GetService <IElasticClient>(); AppSettings settings = app.ApplicationServices.GetService <AppSettings>(); string index = settings != null && settings.ElasticConfig != null && string.IsNullOrWhiteSpace(settings.ElasticConfig.DefaultIndex) ? settings.ElasticConfig.DefaultIndex : "friends"; var observable = Observable.Create <bool> (async o => { var elasticHealth = await client.CatHealthAsync(); if (elasticHealth.IsValid) { Console.WriteLine("Elasticsearch is UP!"); o.OnNext(true); o.OnCompleted(); } else { Console.WriteLine("Elasticsearch is down!"); o.OnError(new Exception("Elasticsearch is down")); } }); var observer = Observer.Create <bool> (isElasticsearchUp => { // Remove then create the index client.DeleteIndex(index); client.CreateIndex(index, c => c.Mappings(ms => ms.Map <Friend> (m => m.AutoMap()))); // Bulk insert random friends; bulk for performance client.Bulk((s) => { return(s.IndexMany(Friend.BuildRandomFriends(10))); }); }); observable.DelaySubscription(TimeSpan.FromSeconds(1)).Retry(60).Subscribe(observer); }
public void Should_Be_Able_To_Index_Corpus_Really_Really_Fast() { var articles = DeserializeCorpus(); var bulkRequest = new BulkRequest(_reutersIndex); var operations = articles.Select(article => new BulkIndexOperation <Article>(article) { Id = article.Id, Index = _reutersIndex }).Cast <IBulkOperation>().ToList(); bulkRequest.Operations = operations; var response = _client.Bulk(bulkRequest); Thread.Sleep(1000); var document = _client.Get <Article>(1, idx => idx.Index(ReutersCorpusIndexName)); document.Should().NotBeNull(); document.Id.Should().Be("1"); }
protected virtual int PerformDocumentIndexing <TDocument>(IElasticClient client, List <TDocument> documents) where TDocument : SearchItemDocumentBase { if (documents.Any()) { var bulkIndexResponse = client.Bulk(b => b .IndexMany(documents, (op, item) => op .Index(this.Name) ) ); if (bulkIndexResponse.Errors) { // Handle error... } return(bulkIndexResponse.Items.Count); } return(0); }
public async Task <BulkResponse> LoadCitiesAsHints() { var section = (await GetHintSections()).FirstOrDefault(); if (section is null) { section = new HintSection("hello world"); await SaveSection(section); } var hints = Cities.GetHints(section); int bulkAmount = 10000; for (var i = 0; hints.Count > i; i += bulkAmount) { var selectedHints = hints.Skip(i).Take(bulkAmount).ToList(); var response = _elasticClient.Bulk(x => x .Index(hintWordIndex) .IndexMany(selectedHints)); } return(null); }
public void PostCatalogsSample() { if (!_elasticClient.Indices.Exists(nameof(IndexCatalog).ToLower()).Exists) { _elasticClient.Indices.Create(nameof(IndexCatalog).ToLower()); } _elasticClient.IndexMany <IndexCatalog>(IndexCatalog.GetCatalog()); var descriptor = new BulkDescriptor(); descriptor.UpdateMany <IndexCatalog>(IndexCatalog.GetCatalog(), (b, u) => b .Index(nameof(IndexCatalog).ToLower()) .Doc(u) .DocAsUpsert()); var insert = _elasticClient.Bulk(descriptor); if (!insert.IsValid) { throw new Exception(insert.OriginalException.ToString()); } }
public void PostLogsSample() { try { var descriptor = new BulkDescriptor(); descriptor.UpdateMany(IndexLog.GetSampleData(), (b, u) => b .Index($"{_elasticIndex}-{DateTime.Now:yyyy.MM.dd}") .Doc(u) .DocAsUpsert()); var insert = _elasticClient.Bulk(descriptor); if (!insert.IsValid) { throw new Exception(insert.OriginalException.ToString()); } } catch (Exception) { throw; } }
public void PostSampleElastic() { var descriptor = new BulkDescriptor(); if (!_elasticClient.Indices.Exists(nameof(IndexActorsModel).ToLower()).Exists) { _elasticClient.Indices.Create(nameof(IndexActorsModel).ToLower()); } _elasticClient.IndexMany(IndexActorsModel.GetSampleData()); //or descriptor.UpdateMany <IndexActorsModel>(IndexActorsModel.GetSampleData(), (b, u) => b .Index(nameof(IndexActorsModel).ToLower()) .Doc(u) .DocAsUpsert()); var insert = _elasticClient.Bulk(descriptor); if (!insert.IsValid) { throw new Exception(insert.OriginalException.ToString()); } }
public void Seed() { var create = _client.CreateIndex(_index, c => c .Settings(s => s .NumberOfReplicas(0) .NumberOfShards(1) ) .Aliases(a => a .Alias(AliasFor <King>(), AliasFilterFor <King>) .Alias(AliasFor <Prince>(), AliasFilterFor <Prince>) .Alias(AliasFor <Duke>(), AliasFilterFor <Duke>) .Alias(AliasFor <Earl>(), AliasFilterFor <Earl>) .Alias(AliasFor <Baron>(), AliasFilterFor <Baron>) ) .Mappings(map => map .Map <King>(RoyalType, m => m .AutoMap() .Properties(props => RoyalProps(props) .Nested <King>(n => n.Name(p => p.Foes).AutoMap()) .Join(j => j .Name(p => p.Join) .Relations(r => r .Join <King, Prince>() .Join <Prince, Duke>() .Join <Duke, Earl>() .Join <Earl, Baron>() ) ) ) ) ) ); var kings = King.Generator.Generate(2) .Select(k => { var foes = King.Generator.Generate(2) .Select(f => { f.Join = null; return(f); }) .ToList(); k.Foes = foes; return(k); }); var bulk = new BulkDescriptor(); IndexAll(bulk, () => kings, king => IndexAll(bulk, () => Prince.Generator.Generate(2), king, prince => IndexAll(bulk, () => Duke.Generator.Generate(3), prince, duke => IndexAll(bulk, () => Earl.Generator.Generate(5), duke, earl => IndexAll(bulk, () => Baron.Generator.Generate(1), earl) ) ) ) ); _client.Bulk(bulk); _client.Refresh(_index); }
public static void LoadData(IElasticClient client) { var r = client.CreateIndex("entities", c => c .AddMapping<JsonObject>(m => m .IdField(i => i.SetIndex("not_analyzed")) .TypeName("locations") .Properties(p => p .String(s => s.Name("id")) .String(s => s.Name("name").Index(FieldIndexOption.analyzed).IndexAnalyzer("standard")) .String(s => s.Name("parentId"))) )); var all = new List<JsonObject>(); var reader = new StreamReader(File.OpenRead(@"c:\temp\countries.csv"), new UTF8Encoding()); while (!reader.EndOfStream) { var line = reader.ReadLine(); if (line == null) continue; var values = line.Split(','); values[2] = values[2].Replace("\"", ""); var location = CreateObject.CreateMiniEntity(values[0], values[1], values[2]); all.Add(location); } var allObjects = all.ToDictionary(json => json.Id); foreach (var item in all) { var path = new List<string>(); if (!String.IsNullOrEmpty(item["parentId"].ToString())) { RecGetParent(path, allObjects, item); path.Reverse(); path.Add(item["name"].ToString()); item.Add("fullPath", String.Join("#.#", path)); } else item.Add("fullPath", String.Empty); } var insertCount = 0; var bulker = new BulkDescriptor(); for (var index = 0; index < all.Count; index++) { var item = all[index]; bulker.Index<JsonObject>(op => op.Index("entities") .Id(Convert.ToString(item["id"])) .Type("locations") .Object(item)); insertCount++; if (insertCount != 1000 && index != (all.Count - 1)) continue; //PushToElastic(client, bulker); var result = client.Bulk(bulker); insertCount = 0; bulker = new BulkDescriptor(); } }
private async Task <int> Transform(int start, int batchSize, string indexName) { var stopwatchIn = new Stopwatch(); stopwatchIn.Start(); var result = await _solrOperation.QueryAsync(SolrQuery.All, new QueryOptions { StartOrCursor = new StartOrCursor.Start(start), Rows = batchSize }); stopwatchIn.Stop(); _logger.LogInformation($" Reader Taked {stopwatchIn.ElapsedMilliseconds} Milliseconds"); result.ForEach(r => { if (r.ContainsKey("AC_GEO")) { var wkt = r["AC_GEO"].ToString().Split(" "); r["AC_GEO"] = new GeoLocation(double.Parse(wkt[1]), double.Parse(wkt[0])); } else { // if(r.ContainsKey("X")&& r.ContainsKey("Y")) // { // r["AC_GEO"] = new GeoLocation(double.Parse(r["Y"].ToString()), double.Parse(r["X"].ToString())); // //Console.WriteLine("以XY坐标合并经纬度"); // } if (r.ContainsKey("AC_XCOORD") && r.ContainsKey("AC_YCOORD")) { r["AC_GEO"] = new GeoLocation(double.Parse(r["AC_YCOORD"].ToString()), double.Parse(r["AC_XCOORD"].ToString())); //Console.WriteLine("以XY坐标合并经纬度"); } else { // Console.WriteLine("缺少经纬度"); } } if (r.ContainsKey("GEO")) { // var geoString = r["GEO"].ToString(); // var geoWKT= GeoWKTReader.Read(); // r["GEO"] = geoWKT; } if (r.ContainsKey("X")) { r.Remove("X"); } if (r.ContainsKey("Y")) { r.Remove("Y"); } if (r.ContainsKey("_version_")) { r.Remove("_version_"); } //return r; }); stopwatchIn.Restart(); var bulkResponse = _elasticClient.Bulk(p => p.IndexMany(result).Index(indexName)); stopwatchIn.Stop(); _logger.LogInformation($" Index Taked {stopwatchIn.ElapsedMilliseconds} Milliseconds"); if (bulkResponse.Errors) { } var flushResponse = _elasticClient.Indices.Flush(); return(result.Count); }
//TODO nullable IndexName and IndexType? /// <summary> /// Shortcut into the Bulk call that deletes the specified objects /// <para> </para>http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html /// </summary> /// <param name="client"></param> /// <typeparam name="T">The type used to infer the default index and typename</typeparam> /// <param name="objects">List of objects to delete</param> /// <param name="index">Override the inferred indexname for T</param> /// <param name="type">Override the inferred typename for T</param> public static IBulkResponse DeleteMany <T>(this IElasticClient client, IEnumerable <T> @objects, string index = null, string type = null) where T : class { var bulkRequest = CreateDeleteBulkRequest(objects, index, type); return(client.Bulk(bulkRequest)); }
public void Import(IEnumerable <T> documents, string index) { var bulk = CreateBulk(documents, index); client.Bulk(bulk); }
public void Import(IEnumerable <T> documents, string indexName) { var bulk = SetupBulk(documents, indexName); _elasticClient.Bulk(bulk); }
private static void BuildIndex(int assinanteId) { var checkIndexOld = client.Indices.Exists(new IndexExistsRequest(Indices.Index(IdxName))); if (checkIndexOld.Exists) { client.Indices.Delete(new DeleteIndexRequest(Indices.Index(IdxName))); } var checkIndex = client.Indices.Exists(new IndexExistsRequest(Indices.Index(IdxName))); if (!checkIndex.Exists) { var createIndexResponse = client.Indices.Create(IdxName, i => i.Map <IndiceDocumento>(m => m.AutoMap())); if (!createIndexResponse.ApiCall.Success || !createIndexResponse.IsValid) { throw new Exception($"Not possible create index.\r\n{createIndexResponse.DebugInformation}.", createIndexResponse.OriginalException); } int PageSize = assinanteId == 23 ? 2000 : 200; int TotalRecords = 0; int TotalPages = 0; using (var connection = new System.Data.SqlClient.SqlConnection(configuration.GetConnectionString("DefaultConnection"))) { if (connection.State != ConnectionState.Open) { connection.Open(); } TotalRecords = connection.ExecuteScalar <int>(sql_count, new { assinanteId }, commandTimeout: 360); TotalPages = (TotalRecords / (assinanteId == 23 ? 2000 : 200)) + 1; if (connection.State != ConnectionState.Closed) { connection.Close(); } Console.WriteLine("Total Pages: " + TotalPages.ToString()); for (int i = 1; i <= TotalPages; i++) { Console.WriteLine("Página: " + i.ToString()); IEnumerable <IndiceDocumento> listIndex = BuilQueryFromDatabase(assinanteId, PageSize, i); var responseBulk = client.Bulk(b => b .Index(IdxName) .IndexMany(listIndex)); if (!responseBulk.ApiCall.Success || !responseBulk.IsValid) { Console.WriteLine("round: " + i.ToString()); Console.WriteLine(responseBulk.DebugInformation); Console.WriteLine(responseBulk.OriginalException); throw new Exception($"Error populating index.\r\n{responseBulk.DebugInformation}.", responseBulk.OriginalException); } } } //foreach (var itemIndex in listIndex) //{ // var response = client.Index(itemIndex, idx => idx.Index(IdxName)); //} } //foreach (var itemIndex in listIndex) //{ // var response = client.Index(itemIndex, idx => idx.Index("docindex")); //} }
public BulkResponse NoRecyclableMemory() => ClientNoRecyclableMemory.Bulk(b => b.IndexMany(Projects));
public IActionResult Populate() { //TODO: Add Lorem ipsum description for full-text search with highlighting and score //Add data var person1 = new PersonFullDetails { Id = 1, Firstname = "Gilles", Lastname = "Lautrou", Age = 30, Bio = "Sites internet, applications métiers, intranets collaboratifs, solutions de mobilité. Webnet rassemble 140 ingénieurs, consultants, experts des technologies internet. Ethique et diversité. 140 experts du digital. Culture de l’innovation. Guidé par vos objectifs.", Roles = new List <string> { "Developer", "Architect", "Manager" }, Company = new PersonFullDetailsCompany { Id = 1, Name = "Webnet" } }; var person2 = new PersonFullDetails { Id = 2, Firstname = "Jean-Pierre", Lastname = "Dupont", Age = 36, Bio = "Microsoft, en tant qu'acteur de la transformation numérique en France, aide les individus et les entreprises du monde entier à exploiter pleinement leur potentiel.", Roles = new List <string> { "Developer" }, Company = new PersonFullDetailsCompany { Id = 1, Name = "Webnet" } }; var person3 = new PersonFullDetails { Id = 3, Firstname = "Claude", Lastname = "Dupont", Age = 42, Bio = "Planifiez plus intelligemment, collaborez mieux et livrez plus rapidement avec Azure DevOps Services, anciennement connu sous le nom Visual Studio Team", Roles = new List <string> { "Developer", "DevOps" }, Company = new PersonFullDetailsCompany { Id = 2, Name = "Microsoft" } }; var data = new[] { person1, person2, person3 }; _elasticClient.Bulk(b => b .IndexMany(data) .Refresh(Refresh.WaitFor)); //var response1 = _elasticClient.IndexDocument(person1); //var response2 = _elasticClient.IndexDocument(person2); //var response3 = _elasticClient.IndexDocument(person3); return(Ok("Populated with some data")); }
public BulkResponse HighLevel() { var lowLevel = Client.Bulk(b => b.IndexMany(Projects)); return(lowLevel); }
public ReindexApiTests(ManualReindexCluster cluster) { _client = cluster.Client; // create a couple of projects var projects = Project.Generator.Generate(2); var indexProjectsResponse = _client.IndexMany(projects, IndexName); _client.Refresh(IndexName); // create a thousand commits and associate with the projects var commits = CommitActivity.Generator.Generate(5000); var bb = new BulkDescriptor(); for (var i = 0; i < commits.Count; i++) { var commit = commits[i]; var project = i % 2 == 0 ? projects[0].Name : projects[1].Name; bb.Index <CommitActivity>(bi => bi .Document(UpdateJoin(commit, project)) .Index(IndexName) .Id(commit.Id) .Routing(project) ); } CommitActivity UpdateJoin(CommitActivity a, string p) { a.ProjectName = p; return(a); } var bulkResult = _client.Bulk(b => bb); bulkResult.ShouldBeValid(); _client.Refresh(IndexName); _reindexManyTypesResult = _client.Reindex <ILazyDocument>(r => r .BackPressureFactor(10) .ScrollAll("1m", 2, s => s .Search(ss => ss .Index(IndexName) .AllTypes() ) .MaxDegreeOfParallelism(4) ) .BulkAll(b => b .Index(NewManyTypesIndexName) .Size(100) .MaxDegreeOfParallelism(2) .RefreshOnCompleted() ) ); _reindexSingleTypeResult = _client.Reindex <Project>(IndexName, NewSingleTypeIndexName); _reindexProjectionResult = _client.Reindex <CommitActivity, CommitActivityVersion2>( IndexName, NewProjectionIndex, p => new CommitActivityVersion2(p)); }
protected override IBulkResponse ExecuteCore(IElasticClient client, string index) { return(client.Bulk(desc => BuildQueryCore(desc, index, _refreshOnSave))); }
public BulkResponse NestUpdatedBulk() => Client.Bulk(b => b.IndexMany(Projects));