public void Initialize(string uri = "http://localhost:9200/", bool deleteIndex = false) { _logger.Info("Initialing elastic search with uri: " + uri); var connectionString = new ConnectionSettings( new Uri(uri)) .DefaultIndex(OSM_NAMES_INDEX) .PrettyJson(); _elasticClient = new ElasticClient(connectionString); if (deleteIndex && _elasticClient.IndexExists(OSM_NAMES_INDEX).Exists) { _elasticClient.DeleteIndex(OSM_NAMES_INDEX); } if (deleteIndex && _elasticClient.IndexExists(OSM_HIGHWAYS_INDEX).Exists) { _elasticClient.DeleteIndex(OSM_HIGHWAYS_INDEX); } _elasticClient.CreateIndex(OSM_HIGHWAYS_INDEX, c => c.Mappings( ms => ms.Map<object>(m => m.Properties(ps => ps.GeoShape(g => g.Name("geometry") .Tree(GeoTree.Geohash) .TreeLevels(10) .DistanceErrorPercentage(0.2)))))); _logger.Info("Finished initialing elastic search with uri: " + uri); }
public virtual void ConfigureIndexes(IElasticClient client, IEnumerable<IElasticIndex> indexes = null) { if (indexes == null) indexes = GetIndexes(); foreach (var idx in indexes) { int currentVersion = GetAliasVersion(client, idx.AliasName); IIndicesOperationResponse response = null; var templatedIndex = idx as ITemplatedElasticIndex; if (templatedIndex != null) response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName)); else if (!client.IndexExists(idx.VersionedName).Exists) response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName)); Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(idx.AliasName).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) descriptor.Add(add => add.Index(name).Alias(idx.AliasName)); response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= idx.Version || currentVersion < 1) continue; var reindexWorkItem = new ReindexWorkItem { OldIndex = String.Concat(idx.AliasName, "-v", currentVersion), NewIndex = idx.VersionedName, Alias = idx.AliasName, DeleteOld = true, ParentMaps = idx.GetIndexTypes() .Select(kvp => new ParentMap {Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath}) .Where(m => !String.IsNullOrEmpty(m.ParentPath)) .ToList() }; bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result; // already reindexing if (isReindexing) continue; // enqueue reindex to new version _lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait(); } }
public void ConfigureIndexes(IElasticClient client) { foreach (var index in GetIndexes()) { IIndicesOperationResponse response = null; int currentVersion = GetAliasVersion(client, index.Name); var templatedIndex = index as ITemplatedElasticSeachIndex; if (templatedIndex != null) response = client.PutTemplate(index.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(index.Name)); else if (!client.IndexExists(index.VersionedName).Exists) response = client.CreateIndex(index.VersionedName, descriptor => index.CreateIndex(descriptor).AddAlias(index.Name)); Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(index.Name).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(index.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) descriptor.Add(add => add.Index(name).Alias(index.Name)); response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(index.VersionedName).Alias(index.Name))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= index.Version || currentVersion < 1) continue; // upgrade _lockProvider.TryUsingAsync("reindex", async () => { await _workItemQueue.EnqueueAsync(new ReindexWorkItem { OldIndex = String.Concat(index.Name, "-v", currentVersion), NewIndex = index.VersionedName, Alias = index.Name, DeleteOld = true }); }, TimeSpan.Zero, CancellationToken.None); } }
private static void CreateIndex(IElasticClient client, string indexName) { var createIndexResponse = client.CreateIndex(indexName, c => c .Settings(s => s .Analysis(a => a .CharFilters(cf => cf .Mapping("programming_language", mcf => mcf .Mappings( "c# => csharp", "C# => Csharp" ) ) ) .Analyzers(an => an .Custom("vIN", ca => ca .CharFilters("html_strip", "programming_language") .Tokenizer("standard") .Filters("standard", "lowercase", "stop") ) .Custom("buyerTypePrice", ca => ca .CharFilters("programming_language") .Tokenizer("standard") .Filters("standard", "lowercase") ) ) ) ) .Mappings(m => m .Map <Vehicle>(x => x .AutoMap() .Properties(p => p .Text(t => t .Name(n => n.Registration) .Boost(3) ) .Text(t => t .Name(n => n.VIN) .Analyzer("vIN") .Boost(1) ) .Text(t => t .Name(n => n.Make) .Boost(2) ) .Text(t => t .Name(n => n.BuyerTypePrices) .Analyzer("buyerTypePrices") .Boost(2) ) .Nested <BuyerTypePrice>(np => np .AutoMap() .Name(nn => nn.BuyerTypePrices) .Properties(cp => cp .Text(t => t .Name(n => n.BuyerTypeId) .Boost(0.6) ) .Text(t => t .Name(n => n.SalePrice) .Boost(0.5) ) ) ) ) ) ) ); }
public void ConfigureIndexes(IElasticClient client) { foreach (var index in GetIndexes()) { IIndicesOperationResponse response = null; int currentVersion = GetAliasVersion(client, index.Name); var templatedIndex = index as ITemplatedElasticSeachIndex; if (templatedIndex != null) { response = client.PutTemplate(index.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(index.Name)); } else if (!client.IndexExists(index.VersionedName).Exists) { response = client.CreateIndex(index.VersionedName, descriptor => index.CreateIndex(descriptor).AddAlias(index.Name)); } Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(index.Name).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(index.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) { descriptor.Add(add => add.Index(name).Alias(index.Name)); } response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(index.VersionedName).Alias(index.Name))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= index.Version || currentVersion < 1) { continue; } // upgrade _lockProvider.TryUsingAsync("reindex", async() => { await _workItemQueue.EnqueueAsync(new ReindexWorkItem { OldIndex = String.Concat(index.Name, "-v", currentVersion), NewIndex = index.VersionedName, Alias = index.Name, DeleteOld = true }); }, TimeSpan.Zero, CancellationToken.None); } }
private static void CreateIndex(IElasticClient client, string indexName) { var createIndexResponse = client.CreateIndex(indexName, c => c .Settings(s => s .Analysis(a => a .CharFilters(cf => cf .Mapping("programming_language", mcf => mcf .Mappings( "c# => csharp", "C# => Csharp" ) ) ) .Analyzers(an => an .Custom("content", ca => ca .CharFilters("html_strip", "programming_language") .Tokenizer("standard") .Filters("standard", "lowercase", "stop") ) .Custom("categories", ca => ca .CharFilters("programming_language") .Tokenizer("standard") .Filters("standard", "lowercase") ) ) ) ) .Mappings(m => m .Map <Log>(x => x .AutoMap() // .Properties(p => p // .Text(t => t // .Name(n => n.Title) // .Boost(3) // ) // .Text(t => t // .Name(n => n.Content) // .Analyzer("content") // .Boost(1) // ) // .Text(t => t // .Name(n => n.Excerpt) // .Boost(2) // ) // .Text(t => t // .Name(n => n.Categories) // .Analyzer("categories") // .Boost(2) // ) // .Nested<Comment>(np => np // .AutoMap() // .Name(nn => nn.Comments) // .Properties(cp => cp // .Text(t => t // .Name(n => n.Author) // .Boost(0.6) // ) // .Text(t => t // .Name(n => n.Content) // .Boost(0.5) // ) // ) // ) // ) ) ) ); }
public void UsingAutoMap() { /** * Auto mapping can take the pain out of having to define a manual mapping for all properties * on the POCO. In this case we want to index two subclasses into a single index. We call Map * for the base class and then call AutoMap foreach of the types we want it to implement */ var createIndexResponse = _client.CreateIndex("myindex", c => c .Map <Document>(m => m .AutoMap <Company>() // <1> Auto map `Company` using the generic method .AutoMap(typeof(Employee)) // <2> Auto map `Employee` using the non-generic method ) ); /** * This produces the following JSON request */ // json var expected = new { mappings = new { properties = new { birthday = new { type = "date" }, employees = new { properties = new { birthday = new { type = "date" }, employees = new { properties = new { }, type = "object" }, hours = new { type = "long" }, isManager = new { type = "boolean" }, join = new { properties = new { }, type = "object" }, lastName = new { fields = new { keyword = new { ignore_above = 256, type = "keyword" } }, type = "text" }, salary = new { type = "integer" } }, type = "object" }, hours = new { type = "long" }, isManager = new { type = "boolean" }, join = new { properties = new { }, type = "object" }, lastName = new { fields = new { keyword = new { ignore_above = 256, type = "keyword" } }, type = "text" }, name = new { fields = new { keyword = new { ignore_above = 256, type = "keyword" } }, type = "text" }, salary = new { type = "integer" } } } }; // hide Expect(expected).FromRequest(createIndexResponse); }
public static void CreateTestIndex(IElasticClient client, string indexName) { var createIndexResult = client.CreateIndex(indexName, c => c .NumberOfReplicas(ElasticsearchConfiguration.NumberOfReplicas) .NumberOfShards(ElasticsearchConfiguration.NumberOfShards) .AddMapping<ElasticsearchProject>(m => m .MapFromAttributes() .Properties(props => props .String(s => s .Name(p => p.Name) .FieldData(fd => fd.Loading(FieldDataLoading.Eager)) .Fields(fields => fields .String(ss => ss .Name("sort") .Index(FieldIndexOption.NotAnalyzed) ) ) ) .String(s => s .Name(ep => ep.Content) .TermVector(TermVectorOption.WithPositionsOffsetsPayloads) ) ) ) .AddAlias(indexName + "-aliased") .AddMapping<Person>(m => m.MapFromAttributes()) .AddMapping<BoolTerm>(m => m.Properties(pp => pp .String(sm => sm.Name(p => p.Name1).Index(FieldIndexOption.NotAnalyzed)) .String(sm => sm.Name(p => p.Name2).Index(FieldIndexOption.NotAnalyzed)) )) ); createIndexResult.IsValid.Should().BeTrue(); }
private static void CreateIndex(IElasticClient elastic) { var result = elastic.CreateIndex(new IndexName { //Name = _indexName, Type = typeof (BlogPost) }, ci => ci //.Index(_indexName) .Mappings(ms => ms .Map<BlogPost>(m => m.AutoMap()) .Map<Author>(m => m.AutoMap()) )); Console.WriteLine(result.ApiCall.Success); }
protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values) { var create = client.CreateIndex(values.FixedForAllCallsValue, c => c.Mappings(map => map.Map <ScriptDocument>(m => m.AutoMap()))); create.ShouldBeValid(); }
private static void ConfigureMapping(IElasticClient searchclient, bool deleteExistingIndexes = false) { if (deleteExistingIndexes) { searchclient.DeleteIndex(i => i.AllIndices()); } if (!searchclient.IndexExists(new IndexExistsRequest(new IndexNameMarker { Name = ElasticSearchRepository <Stack> .StacksIndexName })).Exists) { searchclient.CreateIndex(ElasticSearchRepository <Stack> .StacksIndexName, d => d .AddAlias("stacks") .AddMapping <Stack>(map => map .Dynamic(DynamicMappingOption.Ignore) .IncludeInAll(false) .Properties(p => p .String(f => f.Name(s => s.OrganizationId).IndexName("organization").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(s => s.ProjectId).IndexName("project").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(s => s.SignatureHash).IndexName("signature").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.Type).IndexName("type").Index(FieldIndexOption.NotAnalyzed)) .Date(f => f.Name(s => s.FirstOccurrence).IndexName("first")) .Date(f => f.Name(s => s.LastOccurrence).IndexName("last")) .String(f => f.Name(s => s.Title).IndexName("title").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.1)) .String(f => f.Name(s => s.Description).IndexName("description").Index(FieldIndexOption.Analyzed).IncludeInAll()) .String(f => f.Name(s => s.Tags).IndexName("tag").Index(FieldIndexOption.NotAnalyzed).IncludeInAll().Boost(1.2)) .String(f => f.Name(s => s.References).IndexName("references").Index(FieldIndexOption.Analyzed).IncludeInAll()) .Date(f => f.Name(s => s.DateFixed).IndexName("fixed")) .Boolean(f => f.Name(s => s.IsHidden).IndexName("hidden")) .Boolean(f => f.Name(s => s.IsRegressed).IndexName("regressed")) .Boolean(f => f.Name(s => s.OccurrencesAreCritical).IndexName("critical")) .Number(f => f.Name(s => s.TotalOccurrences).IndexName("occurrences")) ) ) ); } searchclient.PutTemplate(ElasticSearchRepository <PersistentEvent> .EventsIndexName, d => d .Template(ElasticSearchRepository <PersistentEvent> .EventsIndexName + "-*") .AddMapping <PersistentEvent>(map => map .Dynamic(DynamicMappingOption.Ignore) .IncludeInAll(false) .DisableSizeField(false) .Properties(p => p .String(f => f.Name(e => e.OrganizationId).IndexName("organization").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.ProjectId).IndexName("project").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.StackId).IndexName("stack").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.ReferenceId).IndexName("reference").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.SessionId).IndexName("session").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.Type).IndexName("type").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.Source).IndexName("source").Index(FieldIndexOption.NotAnalyzed).IncludeInAll()) .Date(f => f.Name(e => e.Date).IndexName("date")) .String(f => f.Name(e => e.Message).IndexName("message").Index(FieldIndexOption.Analyzed).IncludeInAll()) .String(f => f.Name(e => e.Tags).IndexName("tag").Index(FieldIndexOption.NotAnalyzed).IncludeInAll().Boost(1.1)) .Boolean(f => f.Name(e => e.IsFirstOccurrence).IndexName("first")) .Boolean(f => f.Name(e => e.IsFixed).IndexName("fixed")) .Boolean(f => f.Name(e => e.IsHidden).IndexName("hidden")) .Object <DataDictionary>(f => f.Name(e => e.Data).Properties(p2 => p2 .String(f2 => f2.Name(Event.KnownDataKeys.Version).Index(FieldIndexOption.NotAnalyzed)) .Object <RequestInfo>(f2 => f2.Name(Event.KnownDataKeys.RequestInfo).Properties(p3 => p3 .String(f3 => f3.Name(r => r.ClientIpAddress).IndexName("ip").Index(FieldIndexOption.Analyzed).IncludeInAll()))) .Object <Error>(f2 => f2.Name(Event.KnownDataKeys.Error).Properties(p3 => p3 .String(f3 => f3.Name(r => r.Type).Index(FieldIndexOption.Analyzed).IncludeInAll()))) .Object <EnvironmentInfo>(f2 => f2.Name(Event.KnownDataKeys.EnvironmentInfo).Properties(p3 => p3 .String(f3 => f3.Name(r => r.MachineName).Index(FieldIndexOption.Analyzed).IncludeInAll()))) .Object <UserInfo>(f2 => f2.Name(Event.KnownDataKeys.UserInfo).Properties(p3 => p3 .String(f3 => f3.Name(r => r.Identity).Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.1)))))) ) ) ); }
public virtual void ConfigureIndexes(IElasticClient client, IEnumerable <IElasticIndex> indexes = null) { if (indexes == null) { indexes = GetIndexes(); } foreach (var idx in indexes) { int currentVersion = GetAliasVersion(client, idx.AliasName); IIndicesOperationResponse response = null; var templatedIndex = idx as ITemplatedElasticIndex; if (templatedIndex != null) { response = client.PutTemplate(idx.VersionedName, template => templatedIndex.CreateTemplate(template).AddAlias(idx.AliasName)); } else if (!client.IndexExists(idx.VersionedName).Exists) { response = client.CreateIndex(idx.VersionedName, descriptor => idx.CreateIndex(descriptor).AddAlias(idx.AliasName)); } Debug.Assert(response == null || response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the index or template."); // Add existing indexes to the alias. if (!client.AliasExists(idx.AliasName).Exists) { if (templatedIndex != null) { var indices = client.IndicesStats().Indices.Where(kvp => kvp.Key.StartsWith(idx.VersionedName)).Select(kvp => kvp.Key).ToList(); if (indices.Count > 0) { var descriptor = new AliasDescriptor(); foreach (string name in indices) { descriptor.Add(add => add.Index(name).Alias(idx.AliasName)); } response = client.Alias(descriptor); } } else { response = client.Alias(a => a.Add(add => add.Index(idx.VersionedName).Alias(idx.AliasName))); } Debug.Assert(response != null && response.IsValid, response?.ServerError != null ? response.ServerError.Error : "An error occurred creating the alias."); } // already on current version if (currentVersion >= idx.Version || currentVersion < 1) { continue; } var reindexWorkItem = new ReindexWorkItem { OldIndex = String.Concat(idx.AliasName, "-v", currentVersion), NewIndex = idx.VersionedName, Alias = idx.AliasName, DeleteOld = true, ParentMaps = idx.GetIndexTypes() .Select(kvp => new ParentMap { Type = kvp.Value.Name, ParentPath = kvp.Value.ParentPath }) .Where(m => !String.IsNullOrEmpty(m.ParentPath)) .ToList() }; bool isReindexing = _lockProvider.IsLockedAsync(String.Concat("reindex:", reindexWorkItem.Alias, reindexWorkItem.OldIndex, reindexWorkItem.NewIndex)).Result; // already reindexing if (isReindexing) { continue; } // enqueue reindex to new version _lockProvider.TryUsingAsync("enqueue-reindex", () => _workItemQueue.EnqueueAsync(reindexWorkItem), TimeSpan.Zero, CancellationToken.None).Wait(); } }
public void UsingAutoMapWithAttributes() { var createIndexResponse = client.CreateIndex("myindex", c => c .Mappings(ms => ms .Map <Company>(m => m.AutoMap()) .Map <Employee>(m => m.AutoMap()) ) ); /** */ // json var expected = new { mappings = new { company = new { properties = new { employees = new { properties = new { birthday = new { format = "MMddyyyy", type = "date" }, empl = new { properties = new {}, type = "nested" }, first_name = new { type = "text" }, isManager = new { null_value = false, store = true, type = "boolean" }, last_name = new { type = "text" }, salary = new { coerce = true, doc_values = false, ignore_malformed = true, type = "float" } }, type = "object", store = false }, name = new { null_value = "null", similarity = "BM25", type = "keyword" }, office_hours = new { type = "text" } } }, employee = new { properties = new { birthday = new { format = "MMddyyyy", type = "date" }, empl = new { properties = new {}, type = "nested" }, first_name = new { type = "text" }, isManager = new { null_value = false, store = true, type = "boolean" }, last_name = new { type = "text" }, salary = new { coerce = true, doc_values = false, ignore_malformed = true, type = "float" } } } } }; // hide Expect(expected).NoRoundTrip().WhenSerializing(Encoding.UTF8.GetString(createIndexResponse.ApiCall.RequestBodyInBytes)); }
protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values) { if (client.IndexExists(Index).Exists) { return; } var createIndexResponse = client.CreateIndex(Index, c => c .Settings(s => s .NumberOfShards(1) .NumberOfReplicas(0) ) .Mappings(m => m .Map <Domain.Shape>(mm => mm .AutoMap() .Properties(p => p .GeoShape(g => g .Name(n => n.GeometryCollection) ) .GeoShape(g => g .Name(n => n.Envelope) ) .GeoShape(g => g .Name(n => n.Circle) ) ) ) ) ); if (!createIndexResponse.IsValid) { throw new Exception($"Error creating index for integration test: {createIndexResponse.DebugInformation}"); } var bulk = new List <object>(); // use the low level client to force WKT var typeName = this.Client.Infer.TypeName <Domain.Shape>(); foreach (var shape in Domain.Shape.Shapes) { bulk.Add(new { index = new { _index = Index, _type = typeName, _id = shape.Id } }); bulk.Add(new { id = shape.Id, geometryCollection = GeoWKTWriter.Write(shape.GeometryCollection), envelope = GeoWKTWriter.Write(shape.Envelope), circle = shape.Circle }); } var bulkResponse = this.Client.LowLevel.Bulk <BulkResponse>( PostData.MultiJson(bulk), new BulkRequestParameters { Refresh = Refresh.WaitFor } ); if (!bulkResponse.IsValid) { throw new Exception($"Error indexing shapes for integration test: {bulkResponse.DebugInformation}"); } }
public void MappingManually() { /**==== Manual mapping * To create a mapping for our Company type, we can use the fluent API * and map each property explicitly */ var createIndexResponse = client.CreateIndex("myindex", c => c .Mappings(ms => ms .Map <Company>(m => m .Properties(ps => ps .Text(s => s .Name(n => n.Name) ) .Object <Employee>(o => o .Name(n => n.Employees) .Properties(eps => eps .Text(s => s .Name(e => e.FirstName) ) .Text(s => s .Name(e => e.LastName) ) .Number(n => n .Name(e => e.Salary) .Type(NumberType.Integer) ) ) ) ) ) ) ); /** * Here, the Name property of the `Company` type has been mapped as a {ref_current}/text.html[text datatype] and * the `Employees` property mapped as an {ref_current}/object.html[object datatype]. Within this object mapping, * only the `FirstName`, `LastName` and `Salary` properties of the `Employee` type have been mapped. * * The json mapping for this example looks like */ //json var expected = new { mappings = new { company = new { properties = new { name = new { type = "text" }, employees = new { type = "object", properties = new { firstName = new { type = "text" }, lastName = new { type = "text" }, salary = new { type = "integer" } } } } } } }; // hide Expect(expected).NoRoundTrip().WhenSerializing(Encoding.UTF8.GetString(createIndexResponse.ApiCall.RequestBodyInBytes)); }
public void UsingACustomPropertyVisitor() { /** Now we can pass an instance of our custom visitor to `.AutoMap()` */ var createIndexResponse = client.CreateIndex("myindex", c => c .Mappings(ms => ms .Map <Employee>(m => m.AutoMap(new DisableDocValuesPropertyVisitor())) ) ); /** and any time the client maps a property of the POCO (Employee in this example) as a number (INumberProperty) or boolean (IBooleanProperty), * it will apply the transformation defined in each `Visit()` call respectively, which in this example * disables {ref_current}/doc-values.html[doc_values]. */ // json var expected = new { mappings = new { employee = new { properties = new { birthday = new { type = "date" }, employees = new { properties = new { }, type = "object" }, firstName = new { type = "text", fields = new { keyword = new { type = "keyword", ignore_above = 256 } } }, isManager = new { doc_values = false, type = "boolean" }, lastName = new { type = "text", fields = new { keyword = new { type = "keyword", ignore_above = 256 } } }, salary = new { doc_values = false, type = "integer" }, hours = new { doc_values = false, type = "long" } } } } }; // hide Expect(expected).NoRoundTrip().WhenSerializing(Encoding.UTF8.GetString(createIndexResponse.ApiCall.RequestBodyInBytes)); }
public static void ConfigureMapping(IElasticClient searchclient, bool deleteExistingIndexes = false) { IIndicesOperationResponse response; if (deleteExistingIndexes) { var deleteResponse = searchclient.DeleteIndex(i => i.AllIndices()); Debug.Assert(deleteResponse.IsValid, deleteResponse.ServerError != null ? deleteResponse.ServerError.Error : "An error occurred deleting the indexes."); response = searchclient.DeleteTemplate(ElasticSearchRepository <PersistentEvent> .EventsIndexName); Debug.Assert(response.IsValid, response.ServerError != null ? response.ServerError.Error : "An error occurred deleting the event index template."); } if (!searchclient.IndexExists(new IndexExistsRequest(new IndexNameMarker { Name = ElasticSearchRepository <Stack> .StacksIndexName })).Exists) { response = searchclient.CreateIndex(ElasticSearchRepository <Stack> .StacksIndexName, d => d .AddAlias("stacks") .AddMapping <Stack>(map => map .Dynamic(DynamicMappingOption.Ignore) .Transform(t => t.Script(SET_FIXED_SCRIPT).Language(ScriptLang.Groovy)) .IncludeInAll(false) .Properties(p => p .String(f => f.Name(e => e.Id).IndexName("id").Index(FieldIndexOption.NotAnalyzed).IncludeInAll()) .String(f => f.Name(s => s.OrganizationId).IndexName("organization").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(s => s.ProjectId).IndexName("project").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(s => s.SignatureHash).IndexName("signature").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.Type).IndexName("type").Index(FieldIndexOption.NotAnalyzed)) .Date(f => f.Name(s => s.FirstOccurrence).IndexName("first")) .Date(f => f.Name(s => s.LastOccurrence).IndexName("last")) .String(f => f.Name(s => s.Title).IndexName("title").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.1)) .String(f => f.Name(s => s.Description).IndexName("description").Index(FieldIndexOption.Analyzed).IncludeInAll()) .String(f => f.Name(s => s.Tags).IndexName("tag").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.2)) .String(f => f.Name(s => s.References).IndexName("links").Index(FieldIndexOption.Analyzed).IncludeInAll()) .Date(f => f.Name(s => s.DateFixed).IndexName("fixedon")) .Boolean(f => f.Name("fixed")) .Boolean(f => f.Name(s => s.IsHidden).IndexName("hidden")) .Boolean(f => f.Name(s => s.IsRegressed).IndexName("regressed")) .Boolean(f => f.Name(s => s.OccurrencesAreCritical).IndexName("critical")) .Number(f => f.Name(s => s.TotalOccurrences).IndexName("occurrences")) ) ) ); Debug.Assert(response.IsValid, response.ServerError != null ? response.ServerError.Error : "An error occurred creating the stack index."); } response = searchclient.PutTemplate(ElasticSearchRepository <PersistentEvent> .EventsIndexName, d => d .Template(ElasticSearchRepository <PersistentEvent> .EventsIndexName + "-*") .Settings(s => s.Add("analysis", BuildAnalysisSettings())) .AddMapping <PersistentEvent>(map => map .Dynamic(DynamicMappingOption.Ignore) .IncludeInAll(false) .DisableSizeField(false) .Transform(t => t.Script(FLATTEN_ERRORS_SCRIPT).Language(ScriptLang.Groovy)) .AllField(i => i.IndexAnalyzer("standardplus").SearchAnalyzer("whitespace_lower")) .Properties(p => p .String(f => f.Name(e => e.Id).IndexName("id").Index(FieldIndexOption.NotAnalyzed).IncludeInAll()) .String(f => f.Name(e => e.OrganizationId).IndexName("organization").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.ProjectId).IndexName("project").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.StackId).IndexName("stack").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.ReferenceId).IndexName("reference").Index(FieldIndexOption.Analyzed)) .String(f => f.Name(e => e.SessionId).IndexName("session").Index(FieldIndexOption.Analyzed)) .String(f => f.Name(e => e.Type).IndexName("type").Index(FieldIndexOption.Analyzed)) .String(f => f.Name(e => e.Source).IndexName("source").Index(FieldIndexOption.Analyzed).IncludeInAll()) .Date(f => f.Name(e => e.Date).IndexName("date")) .String(f => f.Name(e => e.Message).IndexName("message").Index(FieldIndexOption.Analyzed).IncludeInAll()) .String(f => f.Name(e => e.Tags).IndexName("tag").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.2)) .GeoPoint(f => f.Name(e => e.Geo).IndexLatLon()) .Number(f => f.Name(e => e.Value).IndexName("value")) .Boolean(f => f.Name(e => e.IsFirstOccurrence).IndexName("first")) .Boolean(f => f.Name(e => e.IsFixed).IndexName("fixed")) .Boolean(f => f.Name(e => e.IsHidden).IndexName("hidden")) .Object <object>(f => f.Name("idx").Dynamic()) .Object <DataDictionary>(f => f.Name(e => e.Data).Path("just_name").Properties(p2 => p2 .String(f2 => f2.Name(Event.KnownDataKeys.Version).IndexName("version").Index(FieldIndexOption.Analyzed).IndexAnalyzer("version_index").SearchAnalyzer("version_search")) .String(f2 => f2.Name(Event.KnownDataKeys.Level).IndexName("level").Index(FieldIndexOption.Analyzed)) .Object <RequestInfo>(f2 => f2.Name(Event.KnownDataKeys.RequestInfo).Path("just_name").Properties(p3 => p3 .String(f3 => f3.Name(r => r.ClientIpAddress).IndexName("ip").Index(FieldIndexOption.Analyzed).IncludeInAll().Analyzer("comma_whitespace")) .String(f3 => f3.Name(r => r.UserAgent).IndexName("useragent").Index(FieldIndexOption.Analyzed)) .String(f3 => f3.Name(r => r.Path).IndexName("path").Index(FieldIndexOption.Analyzed).IncludeInAll()) .Object <DataDictionary>(f3 => f3.Name(e => e.Data).Path("just_name").Properties(p4 => p4 .String(f4 => f4.Name(RequestInfo.KnownDataKeys.Browser).IndexName("browser").Index(FieldIndexOption.Analyzed) .Fields(fields => fields.String(ss => ss.Name("browser.raw").Index(FieldIndexOption.NotAnalyzed)))) .String(f4 => f4.Name(RequestInfo.KnownDataKeys.BrowserVersion).IndexName("browser.version").Index(FieldIndexOption.Analyzed) .Fields(fields => fields.String(ss => ss.Name("browser.version.raw").Index(FieldIndexOption.NotAnalyzed)))) .String(f4 => f4.Name(RequestInfo.KnownDataKeys.BrowserMajorVersion).IndexName("browser.major").Index(FieldIndexOption.NotAnalyzed)) .String(f4 => f4.Name(RequestInfo.KnownDataKeys.Device).IndexName("device").Index(FieldIndexOption.Analyzed) .Fields(fields => fields.String(ss => ss.Name("device.raw").Index(FieldIndexOption.NotAnalyzed)))) .String(f4 => f4.Name(RequestInfo.KnownDataKeys.OS).IndexName("os").Index(FieldIndexOption.Analyzed) .Fields(fields => fields.String(ss => ss.Name("os.raw").Index(FieldIndexOption.NotAnalyzed)))) .String(f4 => f4.Name(RequestInfo.KnownDataKeys.OSVersion).IndexName("os.version").Index(FieldIndexOption.Analyzed) .Fields(fields => fields.String(ss => ss.Name("os.version.raw").Index(FieldIndexOption.NotAnalyzed)))) .String(f4 => f4.Name(RequestInfo.KnownDataKeys.OSMajorVersion).IndexName("os.major").Index(FieldIndexOption.NotAnalyzed)) .Boolean(f4 => f4.Name(RequestInfo.KnownDataKeys.IsBot).IndexName("bot")))))) .Object <Error>(f2 => f2.Name(Event.KnownDataKeys.Error).Path("just_name").Properties(p3 => p3 .String(f3 => f3.Name("all_codes").IndexName("error.code").Index(FieldIndexOption.NotAnalyzed).Analyzer("whitespace").IncludeInAll().Boost(1.1)) .String(f3 => f3.Name("all_messages").IndexName("error.message").Index(FieldIndexOption.Analyzed).IncludeInAll()) .Object <DataDictionary>(f4 => f4.Name(e => e.Data).Path("just_name").Properties(p4 => p4 .Object <object>(f5 => f5.Name(Error.KnownDataKeys.TargetInfo).Path("just_name").Properties(p5 => p5 .String(f6 => f6.Name("ExceptionType").IndexName("error.targettype").Index(FieldIndexOption.Analyzed).IndexAnalyzer("typename").SearchAnalyzer("whitespace_lower").IncludeInAll().Boost(1.2) .Fields(fields => fields.String(ss => ss.Name("error.targettype.raw").Index(FieldIndexOption.NotAnalyzed)))) .String(f6 => f6.Name("Method").IndexName("error.targetmethod").Index(FieldIndexOption.Analyzed).IndexAnalyzer("typename").SearchAnalyzer("whitespace_lower").IncludeInAll().Boost(1.2)))))) .String(f3 => f3.Name("all_types").IndexName("error.type").Index(FieldIndexOption.Analyzed).IndexAnalyzer("typename").SearchAnalyzer("whitespace_lower").IncludeInAll().Boost(1.1)))) .Object <SimpleError>(f2 => f2.Name(Event.KnownDataKeys.SimpleError).Path("just_name").Properties(p3 => p3 .String(f3 => f3.Name("all_messages").IndexName("error.message").Index(FieldIndexOption.Analyzed).IncludeInAll()) .Object <DataDictionary>(f4 => f4.Name(e => e.Data).Path("just_name").Properties(p4 => p4 .Object <object>(f5 => f5.Name(Error.KnownDataKeys.TargetInfo).Path("just_name").Properties(p5 => p5 .String(f6 => f6.Name("ExceptionType").IndexName("error.targettype").Index(FieldIndexOption.Analyzed).IndexAnalyzer("typename").SearchAnalyzer("whitespace_lower").IncludeInAll().Boost(1.2)))))) .String(f3 => f3.Name("all_types").IndexName("error.type").Index(FieldIndexOption.Analyzed).IndexAnalyzer("typename").SearchAnalyzer("whitespace_lower").IncludeInAll().Boost(1.1)))) .Object <EnvironmentInfo>(f2 => f2.Name(Event.KnownDataKeys.EnvironmentInfo).Path("just_name").Properties(p3 => p3 .String(f3 => f3.Name(r => r.IpAddress).IndexName("ip").Index(FieldIndexOption.Analyzed).IncludeInAll().Analyzer("comma_whitespace")) .String(f3 => f3.Name(r => r.MachineName).IndexName("machine").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.1)) .String(f3 => f3.Name(r => r.OSName).IndexName("os").Index(FieldIndexOption.Analyzed)) .String(f3 => f3.Name(r => r.Architecture).IndexName("architecture").Index(FieldIndexOption.NotAnalyzed)))) .Object <UserDescription>(f2 => f2.Name(Event.KnownDataKeys.UserDescription).Path("just_name").Properties(p3 => p3 .String(f3 => f3.Name(r => r.Description).IndexName("user.description").Index(FieldIndexOption.Analyzed).IncludeInAll()) .String(f3 => f3.Name(r => r.EmailAddress).IndexName("user.email").Index(FieldIndexOption.Analyzed).IndexAnalyzer("email").SearchAnalyzer("simple").IncludeInAll().Boost(1.1)))) .Object <UserInfo>(f2 => f2.Name(Event.KnownDataKeys.UserInfo).Path("just_name").Properties(p3 => p3 .String(f3 => f3.Name(r => r.Identity).IndexName("user").Index(FieldIndexOption.Analyzed).IndexAnalyzer("email").SearchAnalyzer("whitespace_lower").IncludeInAll().Boost(1.1)) .String(f3 => f3.Name(r => r.Name).IndexName("user.name").Index(FieldIndexOption.Analyzed).IncludeInAll()))))) ) ) ); Debug.Assert(response.IsValid, response.ServerError != null ? response.ServerError.Error : "An error occurred creating the event index template"); }
public void ElasticSearch_CheckConnection() { var responseDelete = _esClient.DeleteIndex(IndexName); if (!_esClient.IndexExists(IndexName).Exists) { // Create a customize Startswith ICreateIndexRequest Config(CreateIndexDescriptor r) => r.Settings(s => s.NumberOfShards(1) .NumberOfReplicas(5) .Analysis(al => al.Analyzers(a => a.Custom("analyzer_startswith", c => c.Tokenizer("keyword").Filters("lowercase"))))) .Mappings(m => m.Map <Person>(t => t.AutoMap())); var responseCreate = _esClient.CreateIndex(IndexName, Config); //esClient.Map<Person>(m => //{ // var putMappingDescriptor = m.Index(Indices.Index("person")).AutoMap(); // return putMappingDescriptor; //}); } var person = new Person { Id = 1, FirstName = "Võ Kế", LastName = "Nghiệp" }; var repInsertDoc = _esClient.Index(person, s => s.Index <Person>()); person = new Person { Id = 2, FirstName = "Võ Trọng", LastName = "Nghĩa" }; repInsertDoc = _esClient.Index(person, s => s.Index <Person>()); person = new Person { Id = 3, FirstName = "Võ Trắc", LastName = "Nghị" }; repInsertDoc = _esClient.Index(person, s => s.Index <Person>()); person = new Person { Id = 4, FirstName = "Võ Nguyên", LastName = "Khang" }; repInsertDoc = _esClient.Index(person, s => s.Index <Person>()); person = new Person { Id = 5, FirstName = "Delete", LastName = "Delete" }; repInsertDoc = _esClient.Index(person, s => s.Index <Person>()); var repDeleteDoc = _esClient.Delete <Person>(5); person = new Person { Id = 6, FirstName = "Update", LastName = "Update" }; repInsertDoc = _esClient.Index(person, s => s.Index <Person>()); person = new Person { Id = 6, FirstName = "Update 1", LastName = "Update 1" }; repInsertDoc = _esClient.Index(person, s => s.Index <Person>()); var repGetDoc = _esClient.Get <Person>(6); //esClient.DeleteIndex("people"); //Assert.AreEqual(repInsertDoc.Result, Result.Updated); var repUpdateDoc = _esClient.Update <Person, Person>(new Person { Id = 6, FirstName = "Update 1" }, p => p.Doc(new Person { FirstName = "Update 2" })); Assert.AreEqual(repUpdateDoc.Version, 3); }
public void UsingAutoMapWithAttributes() { var createIndexResponse = _client.CreateIndex("myindex", c => c .Mappings(ms => ms .Map <Employee>(m => m.AutoMap()) ) ); /** */ // json var expected = new { mappings = new { employee = new { properties = new { birthday = new { format = "MMddyyyy", type = "date" }, empl = new { properties = new {}, type = "nested" }, first_name = new { type = "text", norms = false, similarity = "LMDirichlet" }, isManager = new { null_value = false, store = true, type = "boolean" }, last_name = new { type = "text" }, office_hours = new { type = "text" }, salary = new { coerce = true, doc_values = false, ignore_malformed = true, type = "float" }, skills = new { properties = new { level = new { type = "byte" }, name = new { type = "text" } }, type = "object" } } } } }; // hide Expect(expected).FromRequest(createIndexResponse); }
private static void ConfigureMapping(IElasticClient searchclient, bool deleteExistingIndexes = false) { if (deleteExistingIndexes) searchclient.DeleteIndex(i => i.AllIndices()); if (!searchclient.IndexExists(new IndexExistsRequest(new IndexNameMarker { Name = ElasticSearchRepository<Stack>.StacksIndexName })).Exists) searchclient.CreateIndex(ElasticSearchRepository<Stack>.StacksIndexName, d => d .AddAlias("stacks") .AddMapping<Stack>(map => map .Dynamic(DynamicMappingOption.Ignore) .IncludeInAll(false) .Properties(p => p .String(f => f.Name(s => s.OrganizationId).IndexName("organization").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(s => s.ProjectId).IndexName("project").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(s => s.SignatureHash).IndexName("signature").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.Type).IndexName("type").Index(FieldIndexOption.NotAnalyzed)) .Date(f => f.Name(s => s.FirstOccurrence).IndexName("first")) .Date(f => f.Name(s => s.LastOccurrence).IndexName("last")) .String(f => f.Name(s => s.Title).IndexName("title").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.1)) .String(f => f.Name(s => s.Description).IndexName("description").Index(FieldIndexOption.Analyzed).IncludeInAll()) .String(f => f.Name(s => s.Tags).IndexName("tag").Index(FieldIndexOption.NotAnalyzed).IncludeInAll().Boost(1.2)) .String(f => f.Name(s => s.References).IndexName("references").Index(FieldIndexOption.Analyzed).IncludeInAll()) .Date(f => f.Name(s => s.DateFixed).IndexName("fixed")) .Boolean(f => f.Name(s => s.IsHidden).IndexName("hidden")) .Boolean(f => f.Name(s => s.IsRegressed).IndexName("regressed")) .Boolean(f => f.Name(s => s.OccurrencesAreCritical).IndexName("critical")) .Number(f => f.Name(s => s.TotalOccurrences).IndexName("occurrences")) ) ) ); searchclient.PutTemplate(ElasticSearchRepository<PersistentEvent>.EventsIndexName, d => d .Template(ElasticSearchRepository<PersistentEvent>.EventsIndexName + "-*") .AddMapping<PersistentEvent>(map => map .Dynamic(DynamicMappingOption.Ignore) .IncludeInAll(false) .DisableSizeField(false) .Properties(p => p .String(f => f.Name(e => e.OrganizationId).IndexName("organization").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.ProjectId).IndexName("project").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.StackId).IndexName("stack").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.ReferenceId).IndexName("reference").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.SessionId).IndexName("session").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.Type).IndexName("type").Index(FieldIndexOption.NotAnalyzed)) .String(f => f.Name(e => e.Source).IndexName("source").Index(FieldIndexOption.NotAnalyzed).IncludeInAll()) .Date(f => f.Name(e => e.Date).IndexName("date")) .String(f => f.Name(e => e.Message).IndexName("message").Index(FieldIndexOption.Analyzed).IncludeInAll()) .String(f => f.Name(e => e.Tags).IndexName("tag").Index(FieldIndexOption.NotAnalyzed).IncludeInAll().Boost(1.1)) .Boolean(f => f.Name(e => e.IsFirstOccurrence).IndexName("first")) .Boolean(f => f.Name(e => e.IsFixed).IndexName("fixed")) .Boolean(f => f.Name(e => e.IsHidden).IndexName("hidden")) .Object<DataDictionary>(f => f.Name(e => e.Data).Properties(p2 => p2 .String(f2 => f2.Name(Event.KnownDataKeys.Version).Index(FieldIndexOption.NotAnalyzed)) .Object<RequestInfo>(f2 => f2.Name(Event.KnownDataKeys.RequestInfo).Properties(p3 => p3 .String(f3 => f3.Name(r => r.ClientIpAddress).IndexName("ip").Index(FieldIndexOption.Analyzed).IncludeInAll()))) .Object<Error>(f2 => f2.Name(Event.KnownDataKeys.Error).Properties(p3 => p3 .String(f3 => f3.Name(r => r.Type).Index(FieldIndexOption.Analyzed).IncludeInAll()))) .Object<EnvironmentInfo>(f2 => f2.Name(Event.KnownDataKeys.EnvironmentInfo).Properties(p3 => p3 .String(f3 => f3.Name(r => r.MachineName).Index(FieldIndexOption.Analyzed).IncludeInAll()))) .Object<UserInfo>(f2 => f2.Name(Event.KnownDataKeys.UserInfo).Properties(p3 => p3 .String(f3 => f3.Name(r => r.Identity).Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.1)))))) ) ) ); }
public static void LoadData(IElasticClient client) { var r = client.CreateIndex("entities", c => c .AddMapping<JsonObject>(m => m .IdField(i => i.SetIndex("not_analyzed")) .TypeName("locations") .Properties(p => p .String(s => s.Name("id")) .String(s => s.Name("name").Index(FieldIndexOption.analyzed).IndexAnalyzer("standard")) .String(s => s.Name("parentId"))) )); var all = new List<JsonObject>(); var reader = new StreamReader(File.OpenRead(@"c:\temp\countries.csv"), new UTF8Encoding()); while (!reader.EndOfStream) { var line = reader.ReadLine(); if (line == null) continue; var values = line.Split(','); values[2] = values[2].Replace("\"", ""); var location = CreateObject.CreateMiniEntity(values[0], values[1], values[2]); all.Add(location); } var allObjects = all.ToDictionary(json => json.Id); foreach (var item in all) { var path = new List<string>(); if (!String.IsNullOrEmpty(item["parentId"].ToString())) { RecGetParent(path, allObjects, item); path.Reverse(); path.Add(item["name"].ToString()); item.Add("fullPath", String.Join("#.#", path)); } else item.Add("fullPath", String.Empty); } var insertCount = 0; var bulker = new BulkDescriptor(); for (var index = 0; index < all.Count; index++) { var item = all[index]; bulker.Index<JsonObject>(op => op.Index("entities") .Id(Convert.ToString(item["id"])) .Type("locations") .Object(item)); insertCount++; if (insertCount != 1000 && index != (all.Count - 1)) continue; //PushToElastic(client, bulker); var result = client.Bulk(bulker); insertCount = 0; bulker = new BulkDescriptor(); } }