protected virtual IndexDocument CreateDocument(string productId, IList <Price> prices) { var document = new IndexDocument(productId); if (prices != null) { foreach (var price in prices) { document.Add(new IndexDocumentField($"price_{price.Currency}_{price.PricelistId}".ToLowerInvariant(), price.EffectiveValue) { IsRetrievable = true, IsFilterable = true }); // Save additional pricing fields for convinient user searches, store price with currency and without one document.Add(new IndexDocumentField($"price_{price.Currency}".ToLowerInvariant(), price.EffectiveValue) { IsRetrievable = true, IsFilterable = true, IsCollection = true }); document.Add(new IndexDocumentField("price", price.EffectiveValue) { IsRetrievable = true, IsFilterable = true, IsCollection = true }); } } document.Add(new IndexDocumentField("is", prices?.Count > 0 ? "priced" : "unpriced") { IsRetrievable = true, IsFilterable = true, IsCollection = true }); return(document); }
protected virtual IndexDocument CreateDocument(Member member) { var document = new IndexDocument(member.Id); document.AddFilterableValue("MemberType", member.MemberType); document.AddFilterableAndSearchableValue("Name", member.Name); document.AddFilterableAndSearchableValues("Emails", member.Emails); document.AddFilterableAndSearchableValues("Phones", member.Phones); document.AddFilterableValues("Groups", member.Groups); document.AddFilterableValue("CreatedDate", member.CreatedDate); document.AddFilterableValue("ModifiedDate", member.ModifiedDate ?? member.CreatedDate); if (member.Addresses?.Any() == true) { foreach (var address in member.Addresses) { IndexAddress(document, address); } } if (member.Notes?.Any() == true) { foreach (var note in member.Notes) { IndexNote(document, note); } } var contact = member as Contact; var employee = member as Employee; var organization = member as Organization; var vendor = member as Vendor; if (contact != null) { IndexContact(document, contact); } else if (employee != null) { IndexEmployee(document, employee); } else if (organization != null) { IndexOrganization(document, organization); } else if (vendor != null) { IndexVendor(document, vendor); } if (!member.DynamicProperties.IsNullOrEmpty()) { foreach (var property in member.DynamicProperties) { IndexDynamicProperty(document, property); } } return(document); }
public static void AddObjectFieldValue <T>(this IndexDocument document, T value) { document.Add(new IndexDocumentField(ObjectFieldName, SerializeObject(value)) { IsRetrievable = true }); }
protected virtual void IndexProductVariation(IndexDocument document, CatalogProduct variation) { if (variation.ProductType == "Physical") { document.Add(new IndexDocumentField("type", "physical") { IsRetrievable = true, IsFilterable = true, IsCollection = true }); IndexIsProperty(document, "physical"); } if (variation.ProductType == "Digital") { document.Add(new IndexDocumentField("type", "digital") { IsRetrievable = true, IsFilterable = true, IsCollection = true }); IndexIsProperty(document, "digital"); } document.Add(new IndexDocumentField("code", variation.Code) { IsRetrievable = true, IsFilterable = true, IsCollection = true }); // add the variation code to content document.Add(new IndexDocumentField("__content", variation.Code) { IsRetrievable = true, IsSearchable = true, IsCollection = true }); IndexCustomProperties(document, variation.Properties, new[] { PropertyType.Variation }); }
protected virtual void AddParentOrganizations(IndexDocument document, ICollection <string> values) { var nonEmptyValues = values?.Where(v => !string.IsNullOrEmpty(v)).ToArray(); document.AddFilterableValues("ParentOrganizations", nonEmptyValues); document.AddFilterableValue("HasParentOrganizations", nonEmptyValues?.Any() ?? false); }
protected virtual void IndexIsProperty(IndexDocument document, string value) { document.Add(new IndexDocumentField("is", value) { IsRetrievable = true, IsFilterable = true, IsCollection = true }); }
protected virtual void IndexOrganization(IndexDocument document, Organization organization) { document.AddSearchableValue(organization.Description); document.AddFilterableValue("BusinessCategory", organization.BusinessCategory); document.AddFilterableValue("OwnerId", organization.OwnerId); AddParentOrganizations(document, new[] { organization.ParentId }); }
public static void AddObjectFieldValue <T>(this IndexDocument document, T value) { document.Add(new IndexDocumentField(ObjectFieldName, value) { IsRetrievable = false, IsFilterable = false, IsSearchable = false }); }
protected virtual void IndexDynamicProperty(IndexDocument document, DynamicObjectProperty property) { var propertyName = property.Name?.ToLowerInvariant(); if (!string.IsNullOrEmpty(propertyName)) { var isCollection = property.IsDictionary || property.IsArray; IList <object> values; if (!property.IsDictionary) { values = property.Values.Where(x => x.Value != null) .Select(x => x.Value) .ToList(); } else { //add all locales in dictionary to searchIndex values = property.Values.Select(x => x.Value) .Cast <DynamicPropertyDictionaryItem>() .Where(x => !string.IsNullOrEmpty(x.Name)) .Select(x => x.Name) .ToList <object>(); } if (values.Any()) { document.Add(new IndexDocumentField(propertyName, values) { IsRetrievable = true, IsFilterable = true, IsCollection = isCollection }); } } }
protected static IndexDocument CreateDocument(string id, string fieldName) { var result = new IndexDocument(id); result.Add(new IndexDocumentField(fieldName, id)); return(result); }
/* ========================================================================== Operations */ public void AddDocument(IndexDocument document) { var versionId = document.GetIntegerValue(IndexFieldName.VersionId); var storedFields = document.Where(f => f.Store == IndexStoringMode.Yes).ToList(); if (storedFields.Count > 0) { StoredData.Add(new Tuple <int, List <IndexField> >(versionId, storedFields)); } foreach (var field in document) { var fieldName = field.Name; if (!IndexData.TryGetValue(fieldName, out var existingFieldData)) { existingFieldData = new Dictionary <string, List <int> >(); IndexData.Add(fieldName, existingFieldData); } var fieldValues = GetValues(field); foreach (var fieldValue in fieldValues) { if (!existingFieldData.TryGetValue(fieldValue, out var versionIds)) { versionIds = new List <int>(); existingFieldData.Add(fieldValue, versionIds); } versionIds.Add(versionId); } } }
protected virtual void IndexCustomProperties(IndexDocument document, ICollection <Property> properties, ICollection <PropertyValue> propertyValues) { foreach (var propValue in propertyValues.Where(x => x.Value != null)) { var property = properties.FirstOrDefault(p => p.Name.EqualsInvariant(propValue.PropertyName) && p.ValueType == propValue.ValueType); var propertyName = propValue.PropertyName?.ToLowerInvariant(); if (!string.IsNullOrEmpty(propertyName)) { var isCollection = property?.Multivalue == true; switch (propValue.ValueType) { case PropertyValueType.Boolean: case PropertyValueType.DateTime: case PropertyValueType.Number: document.Add(new IndexDocumentField(propertyName, propValue.Value) { IsRetrievable = true, IsFilterable = true, IsCollection = isCollection }); break; case PropertyValueType.LongText: document.Add(new IndexDocumentField(propertyName, propValue.Value.ToString().ToLowerInvariant()) { IsRetrievable = true, IsSearchable = true, IsCollection = isCollection }); break; case PropertyValueType.ShortText: // do not tokenize small values as they will be used for lookups and filters document.Add(new IndexDocumentField(propertyName, propValue.Value.ToString()) { IsRetrievable = true, IsFilterable = true, IsCollection = isCollection }); break; } } // Add value to the searchable content field var contentField = string.Concat("__content", property != null && property.Multilanguage && !string.IsNullOrWhiteSpace(propValue.LanguageCode) ? "_" + propValue.LanguageCode.ToLowerInvariant() : string.Empty); switch (propValue.ValueType) { case PropertyValueType.LongText: case PropertyValueType.ShortText: var stringValue = propValue.Value.ToString(); if (!string.IsNullOrWhiteSpace(stringValue)) // don't index empty values { document.Add(new IndexDocumentField(contentField, stringValue.ToLower()) { IsRetrievable = true, IsSearchable = true, IsCollection = true }); } break; } } }
private async Task SaveIndexDocumentAsync(NodeData nodeData, IndexDocument indexDoc, CancellationToken cancellationToken) { var timestamp = await SaveIndexDocumentAsync(nodeData.VersionId, indexDoc, cancellationToken).ConfigureAwait(false); if (timestamp != 0) { nodeData.VersionTimestamp = timestamp; } }
public void index_document_contains_nuked_attribute() { (from XElement node in IndexDocument.Descendants("wrap") where node.Attribute("name").Value.Equals("pharrell") && node.Attribute("version").Value.Equals("1.0.0.0") && node.Attribute("nuked").Value.Equals("true") select node) .ShouldHaveCountOf(1); }
public static void AddFilterableValue(this IndexDocument document, string name, object value) { if (value != null) { document.Add(new IndexDocumentField(name, value) { IsRetrievable = true, IsFilterable = true }); } }
/// <summary> /// Adds given value to the searchable '__content' field /// </summary> /// <param name="document"></param> /// <param name="value"></param> public static void AddSearchableValue(this IndexDocument document, string value) { if (!string.IsNullOrWhiteSpace(value)) { document.Add(new IndexDocumentField(SearchableFieldName, value) { IsRetrievable = true, IsSearchable = true, IsCollection = true }); } }
private static IndexingResultItem GetIndexingResult(IndexDocument document) { var error = document.Id.StartsWith("bad"); return(new IndexingResultItem { Id = document.Id, Succeeded = !error, ErrorMessage = error ? "Search provider error" : null, }); }
public void Search_WhenSearchingByANonExistentField_ReturnsNoResults() { //Arrange var document = new IndexDocument() { Id = Guid.NewGuid(), Fields = { { "Color", "Red" }, { "Size", "Large" } } }; var document2 = new IndexDocument() { Id = Guid.NewGuid(), Fields = { { "Color", "Red" }, { "Size", "Medium" } } }; var document3 = new IndexDocument() { Id = Guid.NewGuid(), Fields = { { "Color", "Blue" }, { "Size", "Small" } } }; _sut.Index(document); _sut.Index(document2); _sut.Index(document3); var searchOptions = new IndexSearchOptions(); var fieldName = Guid.NewGuid().ToString(); searchOptions.SearchBy(fieldName, new List <string>() { "Small", "Medium" }); //Act var result = _sut.Search(searchOptions); //Assert var documentResults = result.DocumentResults; Assert.IsNotNull(documentResults); Assert.AreEqual(0, documentResults.Count()); }
/// <summary> /// Adds given value to the filterable field with given name and to the searchable '__content' field /// </summary> /// <param name="document"></param> /// <param name="name"></param> /// <param name="value"></param> public static void AddFilterableAndSearchableValue(this IndexDocument document, string name, string value) { if (!string.IsNullOrWhiteSpace(value)) { document.Add(new IndexDocumentField(name, value) { IsRetrievable = true, IsFilterable = true }); document.AddSearchableValue(value); } }
protected virtual SearchDocument ConvertToProviderDocument(IndexDocument document, Properties <IProperties> properties, string documentType) { var result = new SearchDocument { Id = document.Id }; foreach (var field in document.Fields.OrderBy(f => f.Name)) { var fieldName = ElasticSearchHelper.ToElasticFieldName(field.Name); if (result.ContainsKey(fieldName)) { var newValues = new List <object>(); var currentValue = result[fieldName]; var currentValues = currentValue as object[]; if (currentValues != null) { newValues.AddRange(currentValues); } else { newValues.Add(currentValue); } newValues.AddRange(field.Values); result[fieldName] = newValues.ToArray(); } else { var dictionary = properties as IDictionary <PropertyName, IProperty>; if (dictionary != null && !dictionary.ContainsKey(fieldName)) { // Create new property mapping var providerField = CreateProviderField(field, documentType); ConfigureProperty(providerField, field, documentType); properties.Add(fieldName, providerField); } var isCollection = field.IsCollection || field.Values.Count > 1; var point = field.Value as GeoPoint; var value = point != null ? (isCollection ? field.Values.Select(v => ((GeoPoint)v).ToElasticValue()).ToArray() : point.ToElasticValue()) : (isCollection ? field.Values : field.Value); result.Add(fieldName, value); } } return(result); }
public IndexDocument PreProcessDocument(IndexDocument document, string indexName) { if (indexName == ProductCatalogSearchDomains.Categories || indexName == ProductCatalogSearchDomains.Products) { if (document.Tags.All(x => x.Name != _organizationPointer)) { document.Tags.Add(new DocumentTag(_organizationPointer, Guid.Empty)); } } return(document); }
protected virtual void IndexEmployee(IndexDocument document, Employee employee) { document.AddFilterableAndSearchableValue("Salutation", employee.Salutation); document.AddFilterableAndSearchableValue("FullName", employee.FullName); document.AddFilterableAndSearchableValue("FirstName", employee.FirstName); document.AddFilterableAndSearchableValue("MiddleName", employee.MiddleName); document.AddFilterableAndSearchableValue("LastName", employee.LastName); document.AddFilterableValue("BirthDate", employee.BirthDate); AddParentOrganizations(document, employee.Organizations); document.AddFilterableValue("EmployeeType", employee.EmployeeType); document.AddFilterableValue("IsActive", employee.IsActive); }
public void IndexDoc_Security_CannotAddPasswordHash() { var passwordHashFieldName = "PasswordHash"; var passwordHashField = new IndexField(passwordHashFieldName, "31275491872354956198543", IndexingMode.NotAnalyzed, IndexStoringMode.No, IndexTermVector.No); var indexDoc = new IndexDocument { passwordHashField }; Assert.IsFalse(indexDoc.Any(f => f.Name == passwordHashFieldName)); Assert.IsNull(indexDoc.GetStringValue(passwordHashFieldName)); Assert.IsFalse(indexDoc.Fields.ContainsKey(passwordHashFieldName)); }
/* AddDocumentActivity, RebuildActivity */ internal static bool AddDocument(IndexDocument document, VersioningInfo versioning) { var delTerms = versioning.Delete.Select(i => new SnTerm(IndexFieldName.VersionId, i)).ToArray(); var updates = GetUpdates(versioning); if (document != null) { SetDocumentFlags(document, versioning); } IndexingEngine.WriteIndex(delTerms, updates, new[] { document }); return(true); }
protected override IndexDocument CreateDocument(CatalogProduct product) { var document = new IndexDocument(product.Id); // adding an index field for the first letter of name document.Add(new IndexDocumentField(FiltersHelper.FirstLetterField, product.Name.ToUpper()[0]) { IsRetrievable = true, IsFilterable = true //IsSearchable = true }); return(document); }
protected virtual IndexDocument CreateDocument(Category category, string[] tags) { var document = new IndexDocument(category.Id); foreach (var tag in tags) { document.Add(new IndexDocumentField(Constants.UserGroupsFieldName, tag) { IsRetrievable = true, IsFilterable = true, IsCollection = true }); } return(document); }
/* AddDocumentActivity, RebuildActivity */ internal async STT.Task <bool> AddDocumentAsync(IndexDocument document, VersioningInfo versioning, CancellationToken cancellationToken) { var delTerms = versioning.Delete.Select(i => new SnTerm(IndexFieldName.VersionId, i)).ToArray(); var updates = GetUpdates(versioning); if (document != null) { SetDocumentFlags(document, versioning); } await IndexingEngine.WriteIndexAsync(delTerms, updates, new[] { document }, cancellationToken).ConfigureAwait(false); return(true); }
private async Task <string> CreateDocAsync(CancellationToken cancellationToken) { var id = GetId(); var doc = new IndexDocument(); doc.Add(new IndexField("TestDocId", id, IndexingMode.Default, IndexStoringMode.Yes, IndexTermVector.Default)); doc.Add(new IndexField("DocType", "DocumentForValidityTest", IndexingMode.Default, IndexStoringMode.Yes, IndexTermVector.Default)); doc.Add(new IndexField("Time", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), IndexingMode.Default, IndexStoringMode.Yes, IndexTermVector.Default)); var docs = new[] { doc }; await _engine.WriteIndexAsync(null, null, docs, cancellationToken); return(id); }
protected virtual void IndexContact(IndexDocument document, Contact contact) { document.AddFilterableAndSearchableValue("Salutation", contact.Salutation); document.AddFilterableAndSearchableValue("FullName", contact.FullName); document.AddFilterableAndSearchableValue("FirstName", contact.FirstName); document.AddFilterableAndSearchableValue("MiddleName", contact.MiddleName); document.AddFilterableAndSearchableValue("LastName", contact.LastName); document.AddFilterableValue("BirthDate", contact.BirthDate); AddParentOrganizations(document, contact.Organizations); document.AddFilterableValue("TaxpayerId", contact.TaxPayerId); document.AddFilterableValue("PreferredDelivery", contact.PreferredDelivery); document.AddFilterableValue("PreferredCommunication", contact.PreferredCommunication); }
protected virtual SearchDocument ConvertToProviderDocument(IndexDocument document, IList <Field> providerFields, string documentType) { var result = new SearchDocument { Id = document.Id }; document.Fields.Insert(0, new IndexDocumentField(AzureSearchHelper.RawKeyFieldName, document.Id) { IsRetrievable = true, IsFilterable = true }); foreach (var field in document.Fields.OrderBy(f => f.Name)) { var fieldName = AzureSearchHelper.ToAzureFieldName(field.Name); if (result.ContainsKey(fieldName)) { var newValues = new List <object>(); var currentValue = result[fieldName]; if (currentValue is object[] currentValues) { newValues.AddRange(currentValues); } else { newValues.Add(currentValue); } newValues.AddRange(field.Values); result[fieldName] = newValues.ToArray(); } else { var providerField = AddProviderField(documentType, providerFields, fieldName, field); var isCollection = providerField.Type.ToString().StartsWith("Collection("); var point = field.Value as GeoPoint; var value = point != null ? (isCollection ? field.Values.Select(v => ((GeoPoint)v).ToDocumentValue()).ToArray() : point.ToDocumentValue()) : (isCollection ? field.Values : field.Value); result.Add(fieldName, value); } } return(result); }
public void PostToIndex(IndexDocument oDoc, bool bCommit) { updater.PostToIndex(oDoc, bCommit); }
public override void IndexDocument(IndexDocument doc) { IndexQueue.Enqueue(doc); if (threadPool.WorkingThreads == 0) { threadPool.AddWorkItem(new ThreadStart(this.WatchQueue)); } }
protected Document ConvertToLuceneDocument(IndexDocument doc) { Document ldoc = new Document(); this.AddToLuceneDocument(ldoc, FieldType.Keyword, "IndexDocumentID", doc.ID); this.AddToLuceneDocument(ldoc, FieldType.Keyword, "IndexDocumentExpires", doc.Expires); foreach (IndexField field in doc.Fields) { this.AddToLuceneDocument(ldoc, field.Type, field.Name, field.Value, field.Boost); } return ldoc; }
protected IndexDocument TranslateDocument(Document doc) { IndexDocument document = new IndexDocument(); foreach (Field f in doc.Fields()) { FieldType type = FieldType.Keyword; if (f.IsIndexed() && f.IsStored() && f.IsTokenized()) { type = FieldType.Text; } else if (f.IsIndexed() && f.IsStored() && !f.IsTokenized()) { type = FieldType.Keyword; } else if (f.IsIndexed() && !f.IsStored() && f.IsTokenized()) { type = FieldType.TextUnStored; } else if (f.IsIndexed() && !f.IsStored() && !f.IsTokenized()) { type = FieldType.UnStored; } else if (!f.IsIndexed() && f.IsStored()) { type = FieldType.UnIndexed; } document.AddField(type, f.Name(), f.StringValue()); } return document; }
public void PostToIndex(IndexDocument oDoc) { updater.PostToIndex(oDoc); }