예제 #1
0
        public virtual CreateIndexDescriptor CreateIndex(CreateIndexDescriptor idx) {
            const string SET_FIXED_SCRIPT = @"ctx._source['fixed'] = !!ctx._source['date_fixed']";

            return idx.AddMapping<Stack>(map => map
                .Dynamic(DynamicMappingOption.Ignore)
                .Transform(t => t.Script(SET_FIXED_SCRIPT).Language(ScriptLang.Groovy))
                .IncludeInAll(false)
                .Properties(p => p
                    .String(f => f.Name(e => e.Id).IndexName("id").Index(FieldIndexOption.NotAnalyzed).IncludeInAll())
                    .String(f => f.Name(s => s.OrganizationId).IndexName("organization").Index(FieldIndexOption.NotAnalyzed))
                    .String(f => f.Name(s => s.ProjectId).IndexName("project").Index(FieldIndexOption.NotAnalyzed))
                    .String(f => f.Name(s => s.SignatureHash).IndexName("signature").Index(FieldIndexOption.NotAnalyzed))
                    .String(f => f.Name(e => e.Type).IndexName("type").Index(FieldIndexOption.NotAnalyzed))
                    .Date(f => f.Name(s => s.FirstOccurrence).IndexName("first"))
                    .Date(f => f.Name(s => s.LastOccurrence).IndexName("last"))
                    .String(f => f.Name(s => s.Title).IndexName("title").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.1))
                    .String(f => f.Name(s => s.Description).IndexName("description").Index(FieldIndexOption.Analyzed).IncludeInAll())
                    .String(f => f.Name(s => s.Tags).IndexName("tag").Index(FieldIndexOption.Analyzed).IncludeInAll().Boost(1.2))
                    .String(f => f.Name(s => s.References).IndexName("links").Index(FieldIndexOption.Analyzed).IncludeInAll())
                    .Date(f => f.Name(s => s.DateFixed).IndexName("fixedon"))
                    .Boolean(f => f.Name("fixed"))
                    .Boolean(f => f.Name(s => s.IsHidden).IndexName("hidden"))
                    .Boolean(f => f.Name(s => s.IsRegressed).IndexName("regressed"))
                    .Boolean(f => f.Name(s => s.OccurrencesAreCritical).IndexName("critical"))
                    .Number(f => f.Name(s => s.TotalOccurrences).IndexName("occurrences"))
                ));
        }
예제 #2
0
        public virtual CreateIndexDescriptor ConfigureDescriptor(CreateIndexDescriptor idx) {
            idx.AddAlias(Name);

            foreach (var t in IndexTypes)
                t.Configure(idx);

            return idx;
        }
        private void Reindex(IObserver <IReindexResponse <T> > observer)
        {
            var fromIndex = this._reindexDescriptor._FromIndexName;
            var toIndex   = this._reindexDescriptor._ToIndexName;
            var scroll    = this._reindexDescriptor._Scroll ?? "2m";

            fromIndex.ThrowIfNullOrEmpty("fromIndex");
            toIndex.ThrowIfNullOrEmpty("toIndex");

            var indexSettings  = this.CurrentClient.GetIndexSettings(this._reindexDescriptor._FromIndexName);
            var createSettings = new CreateIndexDescriptor(this._connectionSettings)
                                 .InitializeUsing(indexSettings.Settings);
            var createIndexResponse = this.CurrentClient
                                      .CreateIndex(toIndex, (c) =>
            {
                if (this._reindexDescriptor._CreateIndexSelector != null)
                {
                    return(this._reindexDescriptor._CreateIndexSelector(createSettings));
                }
                return(c);
            });

            if (!createIndexResponse.IsValid)
            {
                throw new ReindexException(createIndexResponse.ConnectionStatus);
            }

            var page         = 0;
            var searchResult = this.CurrentClient.Search <T>(
                s => s
                .Index(fromIndex)
                .AllTypes()
                .From(0)
                .Take(100)
                .Query(this._reindexDescriptor._QuerySelector ?? (q => q.MatchAll()))
                .SearchType(SearchType.Scan)
                .Scroll(scroll)
                );

            if (searchResult.Total <= 0)
            {
                throw new ReindexException(searchResult.ConnectionStatus, "index " + fromIndex + " has no documents!");
            }
            IBulkResponse indexResult = null;

            do
            {
                searchResult = this.CurrentClient.Scroll <T>(scroll, searchResult.ScrollId);
                if (searchResult.Documents.HasAny())
                {
                    indexResult = this.IndexSearchResults(searchResult, observer, toIndex, page);
                }
                page++;
            } while (searchResult.IsValid && indexResult != null && indexResult.IsValid && searchResult.Documents.HasAny());


            observer.OnCompleted();
        }
예제 #4
0
 public CreateIndexDescriptor CreateIndex(CreateIndexDescriptor idx) {
     var keywordLowercaseAnalyzer = new CustomAnalyzer { Filter = new List<string> { "lowercase" }, Tokenizer = "keyword" };
     return idx.Analysis(descriptor => descriptor.Analyzers(bases => bases.Add(KEYWORD_LOWERCASE, keywordLowercaseAnalyzer)))
               .AddMapping<Application>(GetApplicationMap)
               .AddMapping<Organization>(GetOrganizationMap)
               .AddMapping<Project>(GetProjectMap)
               .AddMapping<Models.Token>(GetTokenMap)
               .AddMapping<User>(GetUserMap)
               .AddMapping<WebHook>(GetWebHookMap);
 }
        public void CreateIndex(string fileName, int maxItems)
        {
            if (!client.IndexExists(ElasticConfig.IndexName).Exists)
            {
                var indexDescriptor = new CreateIndexDescriptor(ElasticConfig.IndexName)
                    .Mappings(ms => ms
                        .Map<Post>(m => m.AutoMap()));

                client.CreateIndex(ElasticConfig.IndexName, i=> indexDescriptor);
            }

            BulkIndex(HostingEnvironment.MapPath("~/data/" + fileName), maxItems);
        }
		public void IgnoringProperties()
		{
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<CompanyWithAttributesAndPropertiesToIgnore>(m => m
						.AutoMap()
					)
				);

			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							name = new
							{
								type = "string"
							}
						}
					}
				}
			};

			var settings = WithConnectionSettings(s => s
				.InferMappingFor<CompanyWithAttributesAndPropertiesToIgnore>(i => i
					.Ignore(p => p.AnotherPropertyToIgnore)
				)
			);

			settings.Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
		public void OverridingAutoMappedAttributes()
		{
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<CompanyWithAttributes>(m => m
						.AutoMap()
						.Properties(ps => ps
							.Nested<Employee>(n => n
								.Name(c => c.Employees)
							)
						)
					)
					.Map<EmployeeWithAttributes>(m => m
						.AutoMap()
						.TtlField(ttl => ttl
							.Enable()
							.Default("10m")
						)
						.Properties(ps => ps
							.String(s => s
								.Name(e => e.FirstName)
								.Fields(fs => fs
									.String(ss => ss
										.Name("firstNameRaw")
										.Index(FieldIndexOption.NotAnalyzed)
									)
									.TokenCount(t => t
										.Name("length")
										.Analyzer("standard")
									)
								)
							)
							.Number(n => n
								.Name(e => e.Salary)
								.Type(NumberType.Double)
								.IgnoreMalformed(false)
							)
							.Date(d => d
								.Name(e => e.Birthday)
								.Format("MM-dd-yy")
							)
						)
					)
				);

			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							employees = new
							{
								type = "nested"
							},
							name = new
							{
								analyzer = "keyword",
								null_value = "null",
								similarity = "BM25",
								type = "string"
							},
							headOfficeHours = new
							{
								type = "string"
							}
						}
					},
					employee = new
					{
						_ttl = new
						{
							enabled = true,
							@default = "10m"
						},
						properties = new
						{
							birthday = new
							{
								format = "MM-dd-yy",
								type = "date"
							},
							empl = new
							{
								path = "employees",
								properties = new
								{
									birthday = new
									{
										type = "date"
									},
									employees = new
									{
										properties = new { },
										type = "object"
									},
									firstName = new
									{
										type = "string"
									},
									hours = new
									{
										type = "long"
									},
									isManager = new
									{
										type = "boolean"
									},
									lastName = new
									{
										type = "string"
									},
									salary = new
									{
										type = "integer"
									}
								},
								type = "nested"
							},
							firstName = new
							{
								fields = new
								{
									firstNameRaw = new
									{
										index = "not_analyzed",
										type = "string"
									},
									length = new
									{
										type = "token_count",
										analyzer = "standard"
									}
								},
								type = "string"
							},
							isManager = new
							{
								null_value = false,
								store = true,
								type = "boolean"
							},
							lastName = new
							{
								type = "string"
							},
							salary = new
							{
								ignore_malformed = false,
								type = "double"
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
		public void MappingManually()
		{
			/** ## Manual mapping
			 * To create a mapping for our Company type, we can use the fluent API
			 * and map each property explicitly
			 */
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Company>(m => m
						.Properties(ps => ps
							.String(s => s
								.Name(c => c.Name)
							)
							.Object<Employee>(o => o
								.Name(c => c.Employees)
								.Properties(eps => eps
									.String(s => s
										.Name(e => e.FirstName)
									)
									.String(s => s
										.Name(e => e.LastName)
									)
									.Number(n => n
										.Name(e => e.Salary)
										.Type(NumberType.Integer)
									)
								)
							)
						)
					)
				);

			/**
			 * Which is all fine and dandy, and useful for some use cases. However in most cases
			 * this is becomes too cumbersome of an approach, and you simply just want to map *all*
			 * the properties of your POCO in a single go.
			 */
			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							name = new
							{
								type = "string"
							},
							employees = new
							{
								type = "object",
								properties = new
								{
									firstName = new
									{
										type = "string"
									},
									lastName = new
									{
										type = "string"
									},
									salary = new
									{
										type = "integer"
									}
								}
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
		public void UsingAutoMapWithAttributes()
		{
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<CompanyWithAttributes>(m => m.AutoMap())
					.Map<EmployeeWithAttributes>(m => m.AutoMap())
				);

			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							employees = new
							{
								path = "employees",
								properties = new
								{
									birthday = new
									{
										type = "date"
									},
									employees = new
									{
										properties = new { },
										type = "object"
									},
									firstName = new
									{
										type = "string"
									},
									hours = new
									{
										type = "long"
									},
									isManager = new
									{
										type = "boolean"
									},
									lastName = new
									{
										type = "string"
									},
									salary = new
									{
										type = "integer"
									}
								},
								store = false,
								type = "object"
							},
							name = new
							{
								analyzer = "keyword",
								null_value = "null",
								similarity = "BM25",
								type = "string"
							},
							headOfficeHours = new
							{
								type = "string"
							}
						}
					},
					employee = new
					{
						properties = new
						{
							birthday = new
							{
								format = "MMddyyyy",
								numeric_resolution = "seconds",
								type = "date"
							},
							empl = new
							{
								path = "employees",
								properties = new
								{
									birthday = new
									{
										type = "date"
									},
									employees = new
									{
										properties = new { },
										type = "object"
									},
									firstName = new
									{
										type = "string"
									},
									hours = new
									{
										type = "long"
									},
									isManager = new
									{
										type = "boolean"
									},
									lastName = new
									{
										type = "string"
									},
									salary = new
									{
										type = "integer"
									}
								},
								type = "nested"
							},
							firstName = new
							{
								type = "string"
							},
							isManager = new
							{
								null_value = false,
								store = true,
								type = "boolean"
							},
							lastName = new
							{
								type = "string"
							},
							salary = new
							{
								coerce = true,
								doc_values = false,
								ignore_malformed = true,
								type = "double"
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing(descriptor as ICreateIndexRequest);
		}
		public void OverridingAutoMappedProperties()
		{
			/**
			* Here we are using AutoMap() to automatically map our company type, but then we're
			* overriding our employee property and making it a `nested` type, since by default,
			* AutoMap() will infer objects as `object`.
			*/
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Company>(m => m
						.AutoMap()
						.Properties(ps => ps
							.Nested<Employee>(n => n
								.Name(c => c.Employees)
								.Properties(eps => eps
								// snip
								)
							)
						)
					)
				);

			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							name = new
							{
								type = "string"
							},
							employees = new
							{
								type = "nested",
								properties = new { }
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
		public void UsingACustomPropertyVisitorOnPropertyInfo()
		{
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Employee>(m => m.AutoMap(new EverythingIsAStringPropertyVisitor()))
				);

			var expected = new
			{
				mappings = new
				{
					employee = new
					{
						properties = new
						{
							birthday = new
							{
								type = "string"
							},
							employees = new
							{
								type = "string"
							},
							firstName = new
							{
								type = "string"
							},
							isManager = new
							{
								type = "string"
							},
							lastName = new
							{
								type = "string"
							},
							salary = new
							{
								type = "string"
							}
						}
					}
				}
			};
		}
예제 #12
0
		public void IgnoringProperties()
		{
			/** All of the properties except `Name` have been ignored in the mapping */
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<CompanyWithAttributesAndPropertiesToIgnore>(m => m
						.AutoMap()
					)
				);

			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							name = new
							{
								type = "text",
								fields = new
								{
									keyword = new
									{
										type = "keyword"
							}
						}
					}
				}
					}
				}
			};

			var settings = WithConnectionSettings(s => s
				.InferMappingFor<CompanyWithAttributesAndPropertiesToIgnore>(i => i
					.Ignore(p => p.AnotherPropertyToIgnore)
				)
			);

			settings.Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
예제 #13
0
		public void MappingManually()
		{
			/** === Manual mapping
			 * To create a mapping for our Company type, we can use the fluent API
			 * and map each property explicitly
			 */
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Company>(m => m
						.Properties(ps => ps
							.Text(s => s
								.Name(c => c.Name) //<1> map `Name` as a `string` type
							)
							.Object<Employee>(o => o  //<2> map `Employees` as an `object` type, mapping each of the properties of `Employee`
								.Name(c => c.Employees)
								.Properties(eps => eps
									.Text(s => s
										.Name(e => e.FirstName)
									)
									.Text(s => s
										.Name(e => e.LastName)
									)
									.Number(n => n
										.Name(e => e.Salary)
										.Type(NumberType.Integer)
									)
								)
							)
						)
					)
				);

			/**
			 * This is all fine and dandy and useful for some use cases however in most cases
			 * this can become verbose and wieldy. The majority of the time you simply just want to map *all*
			 * the properties of a POCO in a single go.
			 */
			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							name = new
							{
								type = "text"
							},
							employees = new
							{
								type = "object",
								properties = new
								{
									firstName = new
									{
										type = "text"
									},
									lastName = new
									{
										type = "text"
									},
									salary = new
									{
										type = "integer"
									}
								}
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
예제 #14
0
		public void OverridingAutoMappedProperties()
		{
			/**
			* Here we are using `.AutoMap()` to automatically map our company type, but then we're
			* overriding our employee property and making it a `nested` type, since by default,
			* `.AutoMap()` will infer objects as `object`.
			*/
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Company>(m => m
						.AutoMap()
						.Properties(ps => ps
							.Nested<Employee>(n => n
								.Name(c => c.Employees)
							)
						)
					)
				);

			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							name = new
							{
								type = "text",
								fields = new
								{
									keyword = new
									{
										type = "keyword"
									}
								}
							},
							employees = new
							{
								type = "nested",
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);

			/**
			 * `.AutoMap()` is idempotent; calling it before or after manually
			 * mapped properties will still yield the same results.
			 */
			descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Company>(m => m
						.Properties(ps => ps
							.Nested<Employee>(n => n
								.Name(c => c.Employees)
							)
						)
						.AutoMap()
					)
				);

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
예제 #15
0
 public CreateIndexDescriptor CreateIndex(CreateIndexDescriptor idx) {
     return idx.AddMapping<Employee>(GetEmployeeMap);
 }
		public void ControllingRecursionDepth()
		{
			/** By default, AutoMap() only goes as far as depth 1 */
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<A>(m => m.AutoMap())
				);

			/** Thus we do not map properties on the second occurrence of our Child property */
			var expected = new
			{
				mappings = new
				{
					a = new
					{
						properties = new
						{
							child = new
							{
								properties = new { },
								type = "object"
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);

			/** Now lets specify a maxRecursion of 3 */
			var withMaxRecursionDescriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<A>(m => m.AutoMap(3))
				);

			/** AutoMap() has now mapped three levels of our Child property */
			var expectedWithMaxRecursion = new
			{
				mappings = new
				{
					a = new
					{
						properties = new
						{
							child = new
							{
								type = "object",
								properties = new
								{
									child = new
									{
										type = "object",
										properties = new
										{
											child = new
											{
												type = "object",
												properties = new
												{
													child = new
													{
														type = "object",
														properties = new { }
													}
												}
											}
										}
									}
								}
							}
						}
					}
				}
			};

			Expect(expectedWithMaxRecursion).WhenSerializing((ICreateIndexRequest)withMaxRecursionDescriptor);
		}
		public void UsingACustomPropertyVisitor()
		{
			/** Now we can pass an instance of our custom visitor to AutoMap() */
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Employee>(m => m.AutoMap(new DisableDocValuesPropertyVisitor()))
				);

			/** and anytime it maps a property as a number (INumberProperty) or boolean (IBooleanProperty) 
			 * it will apply the transformation defined in each Visit() respectively, which in this example
			 * disables doc values.
			 */
			var expected = new
			{
				mappings = new
				{
					employee = new
					{
						properties = new
						{
							birthday = new
							{
								type = "date"
							},
							employees = new
							{
								properties = new { },
								type = "object"
							},
							firstName = new
							{
								type = "string"
							},
							isManager = new
							{
								doc_values = false,
								type = "boolean"
							},
							lastName = new
							{
								type = "string"
							},
							salary = new
							{
								doc_values = false,
								type = "integer"
							}
						}
					}
				}
			};
		}
		public void UsingAutoMap()
		{
			/** ## Simple Automapping
			* This is exactly where `AutoMap()` becomes useful. Instead of manually mapping each property, 
			* explicitly, we can instead call `.AutoMap()` for each of our mappings and let NEST do all the work
			*/
			var descriptor = new CreateIndexDescriptor("myindex")
				.Mappings(ms => ms
					.Map<Company>(m => m.AutoMap())
					.Map<Employee>(m => m.AutoMap())
				);

			/**
			* Observe that NEST has inferred the Elasticsearch types based on the CLR type of our POCO properties.  
			* In this example, 
			* - Birthday was mapped as a date, 
			* - Hours was mapped as a long (ticks)
			* - IsManager was mapped as a boolean, 
			* - Salary as an integer 
			* - Employees as an object
			* and the remaining string properties as strings.
			*/
			var expected = new
			{
				mappings = new
				{
					company = new
					{
						properties = new
						{
							employees = new
							{
								properties = new
								{
									birthday = new
									{
										type = "date"
									},
									employees = new
									{
										properties = new { },
										type = "object"
									},
									firstName = new
									{
										type = "string"
									},
									hours = new
									{
										type = "long"
									},
									isManager = new
									{
										type = "boolean"
									},
									lastName = new
									{
										type = "string"
									},
									salary = new
									{
										type = "integer"
									}
								},
								type = "object"
							},
							name = new
							{
								type = "string"
							}
						}
					},
					employee = new
					{
						properties = new
						{
							birthday = new
							{
								type = "date"
							},
							employees = new
							{
								properties = new { },
								type = "object"
							},
							firstName = new
							{
								type = "string"
							},
							hours = new
							{
								type = "long"
							},
							isManager = new
							{
								type = "boolean"
							},
							lastName = new
							{
								type = "string"
							},
							salary = new
							{
								type = "integer"
							}
						}
					}
				}
			};

			Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
		}
예제 #19
0
 public CreateIndexDescriptor CreateIndex(CreateIndexDescriptor idx) {
     throw new NotImplementedException();
 }
        protected void Initialize(bool reinitialize = false)
        {
            if (reinitialize)
            {
                var result = _client.DeleteIndex(Indices.All);
                if(!result.Acknowledged)
                    throw new Exception("Unable to initialize database.");
            }

            var indexExists = _client.IndexExists(DEFAULT_INDEX);

            if (!indexExists.Exists)
            {
                var createIndexDescriptor = new CreateIndexDescriptor(DEFAULT_INDEX)
                .Mappings(mappings => mappings
                    .Map<Stock>(m => m
                        .Properties(p => p
                            .String(s => s
                                .Name(n => n.Id)
                                .Analyzer("keyword")
                            )
                        )
                    )
                    .Map<LockedStock>(m => m
                        .Properties(p => p
                            .String(s => s
                                .Name(n => n.Id)
                                .Analyzer("keyword")
                            )
                        )
                    )
                );

                var createIndexResponse = _client.CreateIndex(createIndexDescriptor);

                if(!createIndexResponse.Acknowledged)
                    throw new Exception("Unable to create index.", createIndexResponse.OriginalException);
            }
        }