protected virtual IndexDocument CreateDocument(string productId, IList <Price> prices)
{
var document = new IndexDocument(productId);
if (prices != null)
{
foreach (var price in prices)
{
document.Add(new IndexDocumentField($"price_{price.Currency}_{price.PricelistId}".ToLowerInvariant(), price.EffectiveValue)
{
IsRetrievable = true, IsFilterable = true
});
// Save additional pricing fields for convinient user searches, store price with currency and without one
document.Add(new IndexDocumentField($"price_{price.Currency}".ToLowerInvariant(), price.EffectiveValue)
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
document.Add(new IndexDocumentField("price", price.EffectiveValue)
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
}
}
document.Add(new IndexDocumentField("is", prices?.Count > 0 ? "priced" : "unpriced")
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
return(document);
}
protected virtual IndexDocument CreateDocument(Member member)
{
var document = new IndexDocument(member.Id);
document.AddFilterableValue("MemberType", member.MemberType);
document.AddFilterableAndSearchableValue("Name", member.Name);
document.AddFilterableAndSearchableValues("Emails", member.Emails);
document.AddFilterableAndSearchableValues("Phones", member.Phones);
document.AddFilterableValues("Groups", member.Groups);
document.AddFilterableValue("CreatedDate", member.CreatedDate);
document.AddFilterableValue("ModifiedDate", member.ModifiedDate ?? member.CreatedDate);
if (member.Addresses?.Any() == true)
{
foreach (var address in member.Addresses)
{
IndexAddress(document, address);
}
}
if (member.Notes?.Any() == true)
{
foreach (var note in member.Notes)
{
IndexNote(document, note);
}
}
var contact = member as Contact;
var employee = member as Employee;
var organization = member as Organization;
var vendor = member as Vendor;
if (contact != null)
{
IndexContact(document, contact);
}
else if (employee != null)
{
IndexEmployee(document, employee);
}
else if (organization != null)
{
IndexOrganization(document, organization);
}
else if (vendor != null)
{
IndexVendor(document, vendor);
}
if (!member.DynamicProperties.IsNullOrEmpty())
{
foreach (var property in member.DynamicProperties)
{
IndexDynamicProperty(document, property);
}
}
return(document);
}
public static void AddObjectFieldValue <T>(this IndexDocument document, T value)
{
document.Add(new IndexDocumentField(ObjectFieldName, SerializeObject(value))
{
IsRetrievable = true
});
}
protected virtual void IndexProductVariation(IndexDocument document, CatalogProduct variation)
{
if (variation.ProductType == "Physical")
{
document.Add(new IndexDocumentField("type", "physical")
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
IndexIsProperty(document, "physical");
}
if (variation.ProductType == "Digital")
{
document.Add(new IndexDocumentField("type", "digital")
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
IndexIsProperty(document, "digital");
}
document.Add(new IndexDocumentField("code", variation.Code)
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
// add the variation code to content
document.Add(new IndexDocumentField("__content", variation.Code)
{
IsRetrievable = true, IsSearchable = true, IsCollection = true
});
IndexCustomProperties(document, variation.Properties, new[] { PropertyType.Variation });
}
protected virtual void AddParentOrganizations(IndexDocument document, ICollection <string> values)
{
var nonEmptyValues = values?.Where(v => !string.IsNullOrEmpty(v)).ToArray();
document.AddFilterableValues("ParentOrganizations", nonEmptyValues);
document.AddFilterableValue("HasParentOrganizations", nonEmptyValues?.Any() ?? false);
}
protected virtual void IndexIsProperty(IndexDocument document, string value)
{
document.Add(new IndexDocumentField("is", value)
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
}
protected virtual void IndexOrganization(IndexDocument document, Organization organization)
{
document.AddSearchableValue(organization.Description);
document.AddFilterableValue("BusinessCategory", organization.BusinessCategory);
document.AddFilterableValue("OwnerId", organization.OwnerId);
AddParentOrganizations(document, new[] { organization.ParentId });
}
public static void AddObjectFieldValue <T>(this IndexDocument document, T value)
{
document.Add(new IndexDocumentField(ObjectFieldName, value)
{
IsRetrievable = false, IsFilterable = false, IsSearchable = false
});
}
protected virtual void IndexDynamicProperty(IndexDocument document, DynamicObjectProperty property)
{
var propertyName = property.Name?.ToLowerInvariant();
if (!string.IsNullOrEmpty(propertyName))
{
var isCollection = property.IsDictionary || property.IsArray;
IList <object> values;
if (!property.IsDictionary)
{
values = property.Values.Where(x => x.Value != null)
.Select(x => x.Value)
.ToList();
}
else
{
//add all locales in dictionary to searchIndex
values = property.Values.Select(x => x.Value)
.Cast <DynamicPropertyDictionaryItem>()
.Where(x => !string.IsNullOrEmpty(x.Name))
.Select(x => x.Name)
.ToList <object>();
}
if (values.Any())
{
document.Add(new IndexDocumentField(propertyName, values)
{
IsRetrievable = true, IsFilterable = true, IsCollection = isCollection
});
}
}
}
protected static IndexDocument CreateDocument(string id, string fieldName)
{
var result = new IndexDocument(id);
result.Add(new IndexDocumentField(fieldName, id));
return(result);
}
/* ========================================================================== Operations */
public void AddDocument(IndexDocument document)
{
var versionId = document.GetIntegerValue(IndexFieldName.VersionId);
var storedFields = document.Where(f => f.Store == IndexStoringMode.Yes).ToList();
if (storedFields.Count > 0)
{
StoredData.Add(new Tuple <int, List <IndexField> >(versionId, storedFields));
}
foreach (var field in document)
{
var fieldName = field.Name;
if (!IndexData.TryGetValue(fieldName, out var existingFieldData))
{
existingFieldData = new Dictionary <string, List <int> >();
IndexData.Add(fieldName, existingFieldData);
}
var fieldValues = GetValues(field);
foreach (var fieldValue in fieldValues)
{
if (!existingFieldData.TryGetValue(fieldValue, out var versionIds))
{
versionIds = new List <int>();
existingFieldData.Add(fieldValue, versionIds);
}
versionIds.Add(versionId);
}
}
}
protected virtual void IndexCustomProperties(IndexDocument document, ICollection <Property> properties, ICollection <PropertyValue> propertyValues)
{
foreach (var propValue in propertyValues.Where(x => x.Value != null))
{
var property = properties.FirstOrDefault(p => p.Name.EqualsInvariant(propValue.PropertyName) && p.ValueType == propValue.ValueType);
var propertyName = propValue.PropertyName?.ToLowerInvariant();
if (!string.IsNullOrEmpty(propertyName))
{
var isCollection = property?.Multivalue == true;
switch (propValue.ValueType)
{
case PropertyValueType.Boolean:
case PropertyValueType.DateTime:
case PropertyValueType.Number:
document.Add(new IndexDocumentField(propertyName, propValue.Value)
{
IsRetrievable = true, IsFilterable = true, IsCollection = isCollection
});
break;
case PropertyValueType.LongText:
document.Add(new IndexDocumentField(propertyName, propValue.Value.ToString().ToLowerInvariant())
{
IsRetrievable = true, IsSearchable = true, IsCollection = isCollection
});
break;
case PropertyValueType.ShortText: // do not tokenize small values as they will be used for lookups and filters
document.Add(new IndexDocumentField(propertyName, propValue.Value.ToString())
{
IsRetrievable = true, IsFilterable = true, IsCollection = isCollection
});
break;
}
}
// Add value to the searchable content field
var contentField = string.Concat("__content", property != null && property.Multilanguage && !string.IsNullOrWhiteSpace(propValue.LanguageCode) ? "_" + propValue.LanguageCode.ToLowerInvariant() : string.Empty);
switch (propValue.ValueType)
{
case PropertyValueType.LongText:
case PropertyValueType.ShortText:
var stringValue = propValue.Value.ToString();
if (!string.IsNullOrWhiteSpace(stringValue)) // don't index empty values
{
document.Add(new IndexDocumentField(contentField, stringValue.ToLower())
{
IsRetrievable = true, IsSearchable = true, IsCollection = true
});
}
break;
}
}
}
private async Task SaveIndexDocumentAsync(NodeData nodeData, IndexDocument indexDoc, CancellationToken cancellationToken)
{
var timestamp = await SaveIndexDocumentAsync(nodeData.VersionId, indexDoc, cancellationToken).ConfigureAwait(false);
if (timestamp != 0)
{
nodeData.VersionTimestamp = timestamp;
}
}
public void index_document_contains_nuked_attribute()
{
(from XElement node in IndexDocument.Descendants("wrap")
where node.Attribute("name").Value.Equals("pharrell") &&
node.Attribute("version").Value.Equals("1.0.0.0") &&
node.Attribute("nuked").Value.Equals("true")
select node)
.ShouldHaveCountOf(1);
}
public static void AddFilterableValue(this IndexDocument document, string name, object value)
{
if (value != null)
{
document.Add(new IndexDocumentField(name, value)
{
IsRetrievable = true, IsFilterable = true
});
}
}
/// <summary>
/// Adds given value to the searchable '__content' field
/// </summary>
/// <param name="document"></param>
/// <param name="value"></param>
public static void AddSearchableValue(this IndexDocument document, string value)
{
if (!string.IsNullOrWhiteSpace(value))
{
document.Add(new IndexDocumentField(SearchableFieldName, value)
{
IsRetrievable = true, IsSearchable = true, IsCollection = true
});
}
}
private static IndexingResultItem GetIndexingResult(IndexDocument document)
{
var error = document.Id.StartsWith("bad");
return(new IndexingResultItem
{
Id = document.Id,
Succeeded = !error,
ErrorMessage = error ? "Search provider error" : null,
});
}
public void Search_WhenSearchingByANonExistentField_ReturnsNoResults()
{
//Arrange
var document = new IndexDocument()
{
Id = Guid.NewGuid(),
Fields =
{
{ "Color", "Red" },
{ "Size", "Large" }
}
};
var document2 = new IndexDocument()
{
Id = Guid.NewGuid(),
Fields =
{
{ "Color", "Red" },
{ "Size", "Medium" }
}
};
var document3 = new IndexDocument()
{
Id = Guid.NewGuid(),
Fields =
{
{ "Color", "Blue" },
{ "Size", "Small" }
}
};
_sut.Index(document);
_sut.Index(document2);
_sut.Index(document3);
var searchOptions = new IndexSearchOptions();
var fieldName = Guid.NewGuid().ToString();
searchOptions.SearchBy(fieldName, new List <string>()
{
"Small", "Medium"
});
//Act
var result = _sut.Search(searchOptions);
//Assert
var documentResults = result.DocumentResults;
Assert.IsNotNull(documentResults);
Assert.AreEqual(0, documentResults.Count());
}
/// <summary>
/// Adds given value to the filterable field with given name and to the searchable '__content' field
/// </summary>
/// <param name="document"></param>
/// <param name="name"></param>
/// <param name="value"></param>
public static void AddFilterableAndSearchableValue(this IndexDocument document, string name, string value)
{
if (!string.IsNullOrWhiteSpace(value))
{
document.Add(new IndexDocumentField(name, value)
{
IsRetrievable = true, IsFilterable = true
});
document.AddSearchableValue(value);
}
}
protected virtual SearchDocument ConvertToProviderDocument(IndexDocument document, Properties <IProperties> properties, string documentType)
{
var result = new SearchDocument {
Id = document.Id
};
foreach (var field in document.Fields.OrderBy(f => f.Name))
{
var fieldName = ElasticSearchHelper.ToElasticFieldName(field.Name);
if (result.ContainsKey(fieldName))
{
var newValues = new List <object>();
var currentValue = result[fieldName];
var currentValues = currentValue as object[];
if (currentValues != null)
{
newValues.AddRange(currentValues);
}
else
{
newValues.Add(currentValue);
}
newValues.AddRange(field.Values);
result[fieldName] = newValues.ToArray();
}
else
{
var dictionary = properties as IDictionary <PropertyName, IProperty>;
if (dictionary != null && !dictionary.ContainsKey(fieldName))
{
// Create new property mapping
var providerField = CreateProviderField(field, documentType);
ConfigureProperty(providerField, field, documentType);
properties.Add(fieldName, providerField);
}
var isCollection = field.IsCollection || field.Values.Count > 1;
var point = field.Value as GeoPoint;
var value = point != null
? (isCollection ? field.Values.Select(v => ((GeoPoint)v).ToElasticValue()).ToArray() : point.ToElasticValue())
: (isCollection ? field.Values : field.Value);
result.Add(fieldName, value);
}
}
return(result);
}
public IndexDocument PreProcessDocument(IndexDocument document, string indexName)
{
if (indexName == ProductCatalogSearchDomains.Categories || indexName == ProductCatalogSearchDomains.Products)
{
if (document.Tags.All(x => x.Name != _organizationPointer))
{
document.Tags.Add(new DocumentTag(_organizationPointer, Guid.Empty));
}
}
return(document);
}
protected virtual void IndexEmployee(IndexDocument document, Employee employee)
{
document.AddFilterableAndSearchableValue("Salutation", employee.Salutation);
document.AddFilterableAndSearchableValue("FullName", employee.FullName);
document.AddFilterableAndSearchableValue("FirstName", employee.FirstName);
document.AddFilterableAndSearchableValue("MiddleName", employee.MiddleName);
document.AddFilterableAndSearchableValue("LastName", employee.LastName);
document.AddFilterableValue("BirthDate", employee.BirthDate);
AddParentOrganizations(document, employee.Organizations);
document.AddFilterableValue("EmployeeType", employee.EmployeeType);
document.AddFilterableValue("IsActive", employee.IsActive);
}
public void IndexDoc_Security_CannotAddPasswordHash()
{
var passwordHashFieldName = "PasswordHash";
var passwordHashField = new IndexField(passwordHashFieldName, "31275491872354956198543",
IndexingMode.NotAnalyzed, IndexStoringMode.No, IndexTermVector.No);
var indexDoc = new IndexDocument {
passwordHashField
};
Assert.IsFalse(indexDoc.Any(f => f.Name == passwordHashFieldName));
Assert.IsNull(indexDoc.GetStringValue(passwordHashFieldName));
Assert.IsFalse(indexDoc.Fields.ContainsKey(passwordHashFieldName));
}
/* AddDocumentActivity, RebuildActivity */
internal static bool AddDocument(IndexDocument document, VersioningInfo versioning)
{
var delTerms = versioning.Delete.Select(i => new SnTerm(IndexFieldName.VersionId, i)).ToArray();
var updates = GetUpdates(versioning);
if (document != null)
{
SetDocumentFlags(document, versioning);
}
IndexingEngine.WriteIndex(delTerms, updates, new[] { document });
return(true);
}
protected override IndexDocument CreateDocument(CatalogProduct product)
{
var document = new IndexDocument(product.Id);
// adding an index field for the first letter of name
document.Add(new IndexDocumentField(FiltersHelper.FirstLetterField, product.Name.ToUpper()[0])
{
IsRetrievable = true,
IsFilterable = true
//IsSearchable = true
});
return(document);
}
protected virtual IndexDocument CreateDocument(Category category, string[] tags)
{
var document = new IndexDocument(category.Id);
foreach (var tag in tags)
{
document.Add(new IndexDocumentField(Constants.UserGroupsFieldName, tag)
{
IsRetrievable = true, IsFilterable = true, IsCollection = true
});
}
return(document);
}
/* AddDocumentActivity, RebuildActivity */
internal async STT.Task <bool> AddDocumentAsync(IndexDocument document, VersioningInfo versioning, CancellationToken cancellationToken)
{
var delTerms = versioning.Delete.Select(i => new SnTerm(IndexFieldName.VersionId, i)).ToArray();
var updates = GetUpdates(versioning);
if (document != null)
{
SetDocumentFlags(document, versioning);
}
await IndexingEngine.WriteIndexAsync(delTerms, updates, new[] { document }, cancellationToken).ConfigureAwait(false);
return(true);
}
private async Task <string> CreateDocAsync(CancellationToken cancellationToken)
{
var id = GetId();
var doc = new IndexDocument();
doc.Add(new IndexField("TestDocId", id, IndexingMode.Default, IndexStoringMode.Yes, IndexTermVector.Default));
doc.Add(new IndexField("DocType", "DocumentForValidityTest", IndexingMode.Default, IndexStoringMode.Yes, IndexTermVector.Default));
doc.Add(new IndexField("Time", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.ffff"), IndexingMode.Default, IndexStoringMode.Yes, IndexTermVector.Default));
var docs = new[] { doc };
await _engine.WriteIndexAsync(null, null, docs, cancellationToken);
return(id);
}
protected virtual void IndexContact(IndexDocument document, Contact contact)
{
document.AddFilterableAndSearchableValue("Salutation", contact.Salutation);
document.AddFilterableAndSearchableValue("FullName", contact.FullName);
document.AddFilterableAndSearchableValue("FirstName", contact.FirstName);
document.AddFilterableAndSearchableValue("MiddleName", contact.MiddleName);
document.AddFilterableAndSearchableValue("LastName", contact.LastName);
document.AddFilterableValue("BirthDate", contact.BirthDate);
AddParentOrganizations(document, contact.Organizations);
document.AddFilterableValue("TaxpayerId", contact.TaxPayerId);
document.AddFilterableValue("PreferredDelivery", contact.PreferredDelivery);
document.AddFilterableValue("PreferredCommunication", contact.PreferredCommunication);
}
protected virtual SearchDocument ConvertToProviderDocument(IndexDocument document, IList <Field> providerFields, string documentType)
{
var result = new SearchDocument {
Id = document.Id
};
document.Fields.Insert(0, new IndexDocumentField(AzureSearchHelper.RawKeyFieldName, document.Id)
{
IsRetrievable = true, IsFilterable = true
});
foreach (var field in document.Fields.OrderBy(f => f.Name))
{
var fieldName = AzureSearchHelper.ToAzureFieldName(field.Name);
if (result.ContainsKey(fieldName))
{
var newValues = new List <object>();
var currentValue = result[fieldName];
if (currentValue is object[] currentValues)
{
newValues.AddRange(currentValues);
}
else
{
newValues.Add(currentValue);
}
newValues.AddRange(field.Values);
result[fieldName] = newValues.ToArray();
}
else
{
var providerField = AddProviderField(documentType, providerFields, fieldName, field);
var isCollection = providerField.Type.ToString().StartsWith("Collection(");
var point = field.Value as GeoPoint;
var value = point != null
? (isCollection ? field.Values.Select(v => ((GeoPoint)v).ToDocumentValue()).ToArray() : point.ToDocumentValue())
: (isCollection ? field.Values : field.Value);
result.Add(fieldName, value);
}
}
return(result);
}
public void PostToIndex(IndexDocument oDoc, bool bCommit)
{
updater.PostToIndex(oDoc, bCommit);
}
public override void IndexDocument(IndexDocument doc)
{
IndexQueue.Enqueue(doc);
if (threadPool.WorkingThreads == 0)
{
threadPool.AddWorkItem(new ThreadStart(this.WatchQueue));
}
}
protected Document ConvertToLuceneDocument(IndexDocument doc)
{
Document ldoc = new Document();
this.AddToLuceneDocument(ldoc, FieldType.Keyword, "IndexDocumentID", doc.ID);
this.AddToLuceneDocument(ldoc, FieldType.Keyword, "IndexDocumentExpires", doc.Expires);
foreach (IndexField field in doc.Fields)
{
this.AddToLuceneDocument(ldoc, field.Type, field.Name, field.Value, field.Boost);
}
return ldoc;
}
protected IndexDocument TranslateDocument(Document doc)
{
IndexDocument document = new IndexDocument();
foreach (Field f in doc.Fields())
{
FieldType type = FieldType.Keyword;
if (f.IsIndexed() && f.IsStored() && f.IsTokenized())
{
type = FieldType.Text;
}
else if (f.IsIndexed() && f.IsStored() && !f.IsTokenized())
{
type = FieldType.Keyword;
}
else if (f.IsIndexed() && !f.IsStored() && f.IsTokenized())
{
type = FieldType.TextUnStored;
}
else if (f.IsIndexed() && !f.IsStored() && !f.IsTokenized())
{
type = FieldType.UnStored;
}
else if (!f.IsIndexed() && f.IsStored())
{
type = FieldType.UnIndexed;
}
document.AddField(type, f.Name(), f.StringValue());
}
return document;
}
public void PostToIndex(IndexDocument oDoc)
{
updater.PostToIndex(oDoc);
}
