public IList<SearchDocument> Search(IList<Filter> searchFilters, IList<SearchSortField> sortFields)
{
if (searchFilters == null || searchFilters.Count == 0)
return new List<SearchDocument>();
var sort = CreateSort(searchFilters, sortFields);
var reader = IndexReader.Open(_directory, true);
var searcher = new IndexSearcher(reader);
var query = CreateQuery(searchFilters);
var hits = searcher.Search(query, new QueryWrapperFilter(query), 100, sort);
var searchResult = new List<SearchDocument>();
foreach (var facet in hits.ScoreDocs)
{
var doc = searcher.Doc(facet.Doc);
var searchDoc = new SearchDocument
{
Fields = new List<SearchDocumentField>()
};
var docFields = doc.GetFields();
foreach (var field in docFields)
{
var value = doc.Get(field.Name);
searchDoc.Fields.Add(new SearchDocumentField {FieldName = field.Name, Value = value});
}
searchResult.Add(searchDoc);
}
return searchResult;
}
public void Save(SearchDocument doc)
{
if (doc == null) throw new ArgumentNullException(nameof(doc));
if (doc.Fields == null || doc.Fields.Count == 0)
return;
var document = new Document();
foreach (var docField in doc.Fields)
{
if (string.IsNullOrWhiteSpace(docField.FieldName))
throw new Exception("Field name cannot be empty");
if (string.IsNullOrWhiteSpace(docField.Value))
continue;
var field = new Field(docField.FieldName, docField.Value, Field.Store.YES, Field.Index.ANALYZED);
document.Add(field);
}
var writer = new IndexWriter(
_directory,
new StandardAnalyzer(global::Lucene.Net.Util.Version.LUCENE_30),
!_created,
IndexWriter.MaxFieldLength.UNLIMITED);
using (writer)
{
writer.AddDocument(document);
}
_created = true;
}
/// <summary>
/// Called when a catalog entry is indexed.
/// We use this method to load the prices for the variants of a product and store
/// the highest variant price on the product for optimal retrieval in the product listing.
/// </summary>
protected override void OnCatalogEntryIndex(ref SearchDocument document, CatalogEntryDto.CatalogEntryRow entry, string language)
{
if (!entry.ClassTypeId.Equals(EntryType.Product))
{
return;
}
UpdateSearchDocument(ref document, entry, language);
}
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddColor()
{
var entry = GetCatalogEntryRow("Product");
var document = new SearchDocument();
_subject.UpdateSearchDocument(ref document, entry, "en");
document["color"].Should().Equals("Green");
}
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddOriginalUSDPrice()
{
var entry = GetCatalogEntryRow("Product");
var document = new SearchDocument();
_subject.UpdateSearchDocument(ref document, entry, "en");
document[IndexingHelper.GetOriginalPriceField(MarketId.Default, Currency.USD)].Value.ToString()
.Should()
.Equals((1000m).ToString(CultureInfo.InvariantCulture));
}
public void Save(string filename, SearchDocument doc)
{
if (doc == null)
return;
var ws = new XmlWriterSettings();
ws.Indent = true;
ws.IndentChars = " ";
ws.CheckCharacters = true;
var serializer = new XmlSerializer(typeof(SearchDocumentDTO), GetUniverse());
using (var writer = XmlWriter.Create(filename, ws))
serializer.Serialize(writer, _assembler.Create(doc));
}
public void UpdateSearchDocument(ref SearchDocument document, CatalogEntryDto.CatalogEntryRow entry, string language)
{
var sw = new Stopwatch();
sw.Start();
var contentLink = _referenceConverter.GetContentLink(entry.Code);
var productContent = _contentLoader.Get<FashionProduct>(contentLink);
var variants = _contentLoader.GetItems(productContent.GetVariants(_relationRepository), CultureInfo.GetCultureInfo(language)).OfType<FashionVariant>().ToList();
AddPrices(document, variants);
AddColors(document, variants);
AddSizes(document, variants);
AddCodes(document, variants);
document.Add(new SearchField("code", productContent.Code, new[] { SearchField.Store.YES, SearchField.IncludeInDefaultSearch.YES }));
document.Add(new SearchField("displayname", productContent.DisplayName));
document.Add(new SearchField("image_url", _assetUrlResolver.GetAssetUrl<IContentImage>(productContent)));
document.Add(new SearchField("content_link", productContent.ContentLink.ToString()));
document.Add(new SearchField("created", productContent.Created.ToString("yyyyMMddhhmmss")));
document.Add(new SearchField("brand", productContent.Brand));
document.Add(new SearchField("top_category_name", GetTopCategory(productContent).DisplayName));
sw.Stop();
_log.Debug(string.Format("Indexing of {0} for {1} took {2}", productContent.Code, language, sw.Elapsed.Milliseconds));
}
protected override void OnCatalogEntryIndex(ref SearchDocument document, CatalogEntryDto.CatalogEntryRow entry, string language)
{
if (entry.ClassTypeId != "Product")
{
return;
}
var result = GetDocument(language, entry.Code).Result;
if (result == null)
{
throw new Exception(String.Format("could not connect to {0}, please make sure site is active",
_url + String.Format("referenceapi/searchdocuments/{0}/{1}", language, entry.Code)));
}
foreach (var field in result.Fields.Where(field => field.Values.Any()))
{
document.Add
(
field.IsDecimal ?
new SearchField(field.Name, Decimal.Parse(field.Values.First(), CultureInfo.InvariantCulture), field.Attributes.ToArray()) :
new SearchField(field.Name, field.Values.First(), field.Attributes.ToArray())
);
}
}
private void DeleteSearchDocumentInternal(SearchDocument searchDocument, bool autoCommit)
{
var query = new BooleanQuery();
if (searchDocument.SearchTypeId > -1)
{
query.Add(NumericValueQuery(Constants.SearchTypeTag, searchDocument.SearchTypeId), Occur.MUST);
}
if (searchDocument.PortalId > -1)
{
query.Add(NumericValueQuery(Constants.PortalIdTag, searchDocument.PortalId), Occur.MUST);
}
if (searchDocument.RoleId > -1)
{
query.Add(NumericValueQuery(Constants.RoleIdTag, searchDocument.RoleId), Occur.MUST);
}
if (searchDocument.ModuleDefId > 0)
{
query.Add(NumericValueQuery(Constants.ModuleDefIdTag, searchDocument.ModuleDefId), Occur.MUST);
}
if (searchDocument.ModuleId > 0)
{
query.Add(NumericValueQuery(Constants.ModuleIdTag, searchDocument.ModuleId), Occur.MUST);
}
if (searchDocument.TabId > 0)
{
query.Add(NumericValueQuery(Constants.TabIdTag, searchDocument.TabId), Occur.MUST);
}
if (searchDocument.AuthorUserId > 0)
{
query.Add(NumericValueQuery(Constants.AuthorIdTag, searchDocument.AuthorUserId), Occur.MUST);
}
if (!string.IsNullOrEmpty(searchDocument.UniqueKey))
{
query.Add(new TermQuery(new Term(Constants.UniqueKeyTag, searchDocument.UniqueKey)), Occur.MUST);
}
if (!string.IsNullOrEmpty(searchDocument.QueryString))
{
query.Add(new TermQuery(new Term(Constants.QueryStringTag, searchDocument.QueryString)), Occur.MUST);
}
if (!string.IsNullOrEmpty(searchDocument.CultureCode))
{
query.Add(NumericValueQuery(Constants.LocaleTag, Localization.GetCultureLanguageID(searchDocument.CultureCode)), Occur.MUST);
}
LuceneController.Instance.Delete(query);
if (autoCommit)
{
this.Commit();
}
}
public void DeleteSearchDocument(SearchDocument searchDocument)
{
this.DeleteSearchDocumentInternal(searchDocument, false);
}
public void GetSearchResultsDetailed()
{
const string keyword = "super";
const string moduleBody = "super content is here";
const string userUrl = "mysite/userid/1";
const string tabUrl1 = "mysite/Home";
const string tabUrl2 = "mysite/AboutUs";
//first tab with 2 modules
var doc1 = new SearchDocument {
UniqueKey = "key01", TabId = TabId1, Url = tabUrl1, Title = keyword, SearchTypeId = TabSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow
};
var doc2 = new SearchDocument {
UniqueKey = "key02", TabId = TabId1, Title = keyword, Url = tabUrl1, SearchTypeId = ModuleSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, ModuleDefId = HtmlModuleDefId, ModuleId = HtmlModuleId2, Body = moduleBody, RoleId = 731
};
var doc3 = new SearchDocument {
UniqueKey = "key03", TabId = TabId1, Title = keyword, Url = tabUrl1, SearchTypeId = ModuleSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, ModuleDefId = HtmlModuleDefId, ModuleId = HtmlModuleId1, Body = moduleBody, RoleId = 731
};
//second tab with 1 module
var doc4 = new SearchDocument {
UniqueKey = "key04", TabId = TabId2, Url = tabUrl2, Title = keyword, SearchTypeId = TabSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, RoleId = RoleId0
};
var doc5 = new SearchDocument {
UniqueKey = "key05", TabId = TabId2, Title = keyword, Url = tabUrl2, SearchTypeId = ModuleSearchTypeId, ModuleDefId = HtmlModuleId, ModuleId = HtmlModuleId3, ModifiedTimeUtc = DateTime.UtcNow, Body = moduleBody, RoleId = 731
};
//user doc
var userdoc = new SearchDocument {
UniqueKey = "key06", Url = userUrl, Title = keyword, SearchTypeId = UserSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, RoleId = RoleId731
};
_internalSearchController.AddSearchDocument(doc1);
_internalSearchController.AddSearchDocument(doc2);
_internalSearchController.AddSearchDocument(doc3);
_internalSearchController.AddSearchDocument(doc4);
_internalSearchController.AddSearchDocument(doc5);
_internalSearchController.AddSearchDocument(userdoc);
var query = new SearchQuery
{
KeyWords = keyword,
SearchTypeIds = new[] { ModuleSearchTypeId, TabSearchTypeId, UserSearchTypeId },
RoleId = 731
};
//Run
var search = GetGroupedDetailViewResults(query);
//Assert
var groupedDetailViews = search as List <GroupedDetailView> ?? search.ToList();
//Overall 3 groups - tab1, tab2 and user
Assert.AreEqual(3, groupedDetailViews.Count());
//Tab 1 has 2 DetailViews
Assert.AreEqual(2, groupedDetailViews.Single(x => x.DocumentUrl == tabUrl1).Results.Count());
//Tab 2 has 1 DetailViews
Assert.AreEqual(1, groupedDetailViews.Single(x => x.DocumentUrl == tabUrl2).Results.Count());
//UserUrl has 1 DetailViews
Assert.AreEqual(1, groupedDetailViews.Single(x => x.DocumentUrl == userUrl).Results.Count());
}
private String DoProductIdx(DotNetNuke.Entities.Portals.PortalInfo portal, DateTime lastrun, Boolean debug)
{
if (debug)
{
InternalSearchController.Instance.DeleteAllDocuments(portal.PortalID, SearchHelper.Instance.GetSearchTypeByName("tab").SearchTypeId);
}
var searchDocs = new List <SearchDocument>();
var culturecodeList = DnnUtils.GetCultureCodeList(portal.PortalID);
var storeSettings = new StoreSettings(portal.PortalID);
foreach (var lang in culturecodeList)
{
var strContent = "";
// select all products
var objCtrl = new NBrightBuyController();
var strFilter = " and NB1.ModifiedDate > convert(datetime,'" + lastrun.ToString("s") + "') ";
if (debug)
{
strFilter = "";
}
var l = objCtrl.GetList(portal.PortalID, -1, "PRD", strFilter);
foreach (var p in l)
{
var prodData = new ProductData(p.ItemID, lang);
strContent = prodData.Info.GetXmlProperty("genxml/textbox/txtproductref") + " : " + prodData.SEODescription + " " + prodData.SEOName + " " + prodData.SEOTagwords + " " + prodData.SEOTitle;
if (strContent != "")
{
var tags = new List <String>();
tags.Add("nbsproduct");
//Get the description string
string strDescription = HtmlUtils.Shorten(HtmlUtils.Clean(strContent, false), 100, "...");
var searchDoc = new SearchDocument();
// Assigns as a Search key the SearchItems'
searchDoc.UniqueKey = prodData.Info.ItemID.ToString("");
searchDoc.QueryString = "ref=" + prodData.Info.GetXmlProperty("genxml/textbox/txtproductref");
searchDoc.Title = prodData.ProductName;
searchDoc.Body = strContent;
searchDoc.Description = strDescription;
if (debug)
{
searchDoc.ModifiedTimeUtc = DateTime.Now.Date;
}
else
{
searchDoc.ModifiedTimeUtc = prodData.Info.ModifiedDate;
}
searchDoc.AuthorUserId = 1;
searchDoc.TabId = storeSettings.ProductDetailTabId;
searchDoc.PortalId = portal.PortalID;
searchDoc.SearchTypeId = SearchHelper.Instance.GetSearchTypeByName("tab").SearchTypeId;
searchDoc.CultureCode = lang;
searchDoc.Tags = tags;
//Add Module MetaData
searchDoc.ModuleDefId = 0;
searchDoc.ModuleId = 0;
searchDocs.Add(searchDoc);
}
}
}
//Index
InternalSearchController.Instance.AddSearchDocuments(searchDocs);
InternalSearchController.Instance.Commit();
return(" - NBS-DNNIDX scheduler ACTIVATED ");
}
public virtual SearchDocument GetDocFromContent(IContent content)
{
try
{
var document = new Document();
IncludeDefaultField(document, content);
var possibleIndexModels = LuceneConfiguration.IncludedTypes.Where(x => x.Value.ContentType.IsAssignableFrom(content.GetOriginalType())).Select(x => x.Value).ToList();
foreach (var documentIndexModel in possibleIndexModels)
{
IncludeContentField(document, content, documentIndexModel);
var computedFieldList = documentIndexModel.IndexedFields;
foreach (var field in computedFieldList)
{
var fieldInstance = field.Value;
var value = fieldInstance.GetValue(content, field.Key);
if (value != null)
{
AbstractField luceneField;
var indexFieldName = ContentIndexHelpers.GetIndexFieldName(field.Key);
var existedFieldName = document.GetField(indexFieldName);
if (existedFieldName != null)
{
document.RemoveField(indexFieldName);
}
if (fieldInstance.DataType == LuceneFieldType.Multilist)
{
var listValue = (List <string>)value;
foreach (var item in listValue)
{
document.Add(new Field(indexFieldName, item, fieldInstance.Store, fieldInstance.Index, fieldInstance.Vector));
}
continue;
}
switch (fieldInstance.DataType)
{
case LuceneFieldType.Datetime:
DateTime d1 = Convert.ToDateTime(value);
luceneField = new Field(indexFieldName, DateTools.DateToString(d1.ToUniversalTime(), DateTools.Resolution.SECOND), fieldInstance.Store, fieldInstance.Index, fieldInstance.Vector);
break;
case LuceneFieldType.Numeric:
luceneField = new NumericField(indexFieldName, fieldInstance.Store, true).SetLongValue(string.IsNullOrEmpty(value + "") ? 0 : long.Parse(value + ""));
break;
default:
luceneField = new Field(indexFieldName, value.ToString(), fieldInstance.Store, fieldInstance.Index, fieldInstance.Vector);
break;
}
document.Add(luceneField);
}
}
}
var result = new SearchDocument()
{
Id = SearchDocument.FormatDocumentId(content),
Document = document
};
return(result);
}
catch (Exception ex)
{
_logger.Error("Fetch Document error", ex);
return(null);
}
}
/// <summary>
/// Parse JSON into a SearchDocument.
/// </summary>
/// <param name="reader">The JSON reader.</param>
/// <param name="typeToConvert">The type to convert to.</param>
/// <param name="options">Serialization options.</param>
/// <param name="recursionDepth">
/// Depth of the current read recursion to bail out of circular
/// references.
/// </param>
/// <returns>A deserialized SearchDocument.</returns>
public static SearchDocument ReadSearchDocument(
ref Utf8JsonReader reader,
Type typeToConvert,
JsonSerializerOptions options,
int?recursionDepth = null)
{
Debug.Assert(typeToConvert != null);
Debug.Assert(typeToConvert.IsAssignableFrom(typeof(SearchDocument)));
recursionDepth ??= options.GetMaxRecursionDepth();
AssertRecursionDepth(recursionDepth.Value);
SearchDocument doc = new SearchDocument();
reader.Expects(JsonTokenType.StartObject);
while (reader.Read() && reader.TokenType != JsonTokenType.EndObject)
{
string propertyName = reader.ExpectsPropertyName();
// Ignore OData properties - we don't expose those on custom
// user schemas
if (!propertyName.StartsWith(Constants.ODataKeyPrefix, StringComparison.OrdinalIgnoreCase))
{
object value = ReadSearchDocObject(ref reader, recursionDepth.Value - 1);
doc[propertyName] = value;
}
else
{
reader.Skip();
}
}
return(doc);
// TODO: #10596 - Investigate using JsonSerializer for reading SearchDocument properties
// The built in converters for JsonSerializer are a little more
// helpful than we want right now and will do things like turn "1"
// to the integer 1 instead of a string. The number of special
// cases needed for converting dynamic documents is small enough
// that we're hard coding them here for now. We'll revisit with
// Search experts and their customer scenarios to get this right in
// the next preview.
object ReadSearchDocObject(ref Utf8JsonReader reader, int depth)
{
AssertRecursionDepth(depth);
switch (reader.TokenType)
{
case JsonTokenType.String:
case JsonTokenType.Number:
case JsonTokenType.True:
case JsonTokenType.False:
case JsonTokenType.None:
case JsonTokenType.Null:
return(ReadPrimitiveValue(ref reader));
case JsonTokenType.StartObject:
// TODO: #10592- Unify on an Azure.Core spatial type
// Return a complex object
return(ReadSearchDocument(ref reader, typeof(SearchDocument), options, depth - 1));
case JsonTokenType.StartArray:
var list = new List <object>();
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
{
recursionDepth--;
object value = ReadSearchDocObject(ref reader, depth - 1);
list.Add(value);
}
return(list.ToArray());
default:
throw new JsonException();
}
}
}
public void SearchDocumentsTest()
{
var locationData = new LocationData(ConfigurationManager.AppSettings[ConfigurationKeys.BingMapsKey]);
var searchServiceName = ConfigurationManager.AppSettings[ConfigurationKeys.SearchServiceName];
var searchServiceKey = ConfigurationManager.AppSettings[ConfigurationKeys.SearchServiceKey];
var searchClient = new SearchDocument(searchServiceName, searchServiceKey);
var data = locationData.GetBoundingBoxCoordinates("Delhi,India").Result;
var searchResult = searchClient.SearchDocuments <GpsSensorRecord>("*", SearchDocument.FilterTextForLocationBounds("geoCoordinates", data));
var locationPoint = new LocationPoint {
Latitude = 28.644979476928711, Longitude = 77.2164306640625
};
var searchByDistance = searchClient.SearchDocuments <GpsSensorRecord>("*", SearchDocument.FilterTextForDistanceFromPoint("geoCoordinates", locationPoint, 100));
Assert.IsTrue(searchResult.Any());
Assert.IsTrue(searchByDistance.Any());
var dataUsa = locationData.GetBoundingBoxCoordinates("Washington,USA").Result;
var searchResultUsa = searchClient.SearchDocuments <GpsSensorRecord>("*", SearchDocument.FilterTextForLocationBounds("geoCoordinates", dataUsa));
Assert.IsTrue(searchResultUsa.Any());
}
/// <summary>
/// Geocodes and returns the result.
/// </summary>
/// <param name="street"></param>
/// <param name="houseNumber"></param>
/// <param name="country"></param>
/// <param name="postalCode"></param>
/// <param name="commune"></param>
/// <returns></returns>
public IGeoCoderResult Code(
string country,
string postalCode,
string commune,
string street,
string houseNumber)
{
// build the request url.
var builder = new StringBuilder();
builder.Append(street);
builder.Append(" ");
builder.Append(houseNumber);
builder.Append(" ");
builder.Append(postalCode);
builder.Append(" ");
builder.Append(commune);
builder.Append(" ");
builder.Append(country);
builder.Append(" ");
string url = string.Format(System.Globalization.CultureInfo.InvariantCulture,
_geocodingUrl + "&format=xml&polygon=1&addressdetails=1", builder);
// create the source and get the xml.
IXmlSource source = this.DownloadXml(url);
// create the kml.
var search_doc = new SearchDocument(source);
// check if there are responses.
var res = new GeoCoderResult();
res.Accuracy = AccuracyEnum.UnkownLocationLevel;
if (search_doc.Search is searchresults)
{
searchresults result_v1 = search_doc.Search as searchresults;
if (result_v1.place != null && result_v1.place.Length > 0)
{
double latitude;
double longitude;
if (double.TryParse(result_v1.place[0].lat, System.Globalization.NumberStyles.Float,
System.Globalization.CultureInfo.InvariantCulture, out latitude)
&&
double.TryParse(result_v1.place[0].lon, System.Globalization.NumberStyles.Float,
System.Globalization.CultureInfo.InvariantCulture, out longitude))
{
res.Latitude = latitude;
res.Longitude = longitude;
res.Text = result_v1.place[0].display_name;
switch (result_v1.place[0].@class)
{
case "place":
switch (result_v1.place[0].type)
{
case "town":
res.Accuracy = AccuracyEnum.TownLevel;
break;
case "house":
res.Accuracy = AccuracyEnum.AddressLevel;
break;
}
break;
case "highway":
res.Accuracy = AccuracyEnum.StreetLevel;
break;
case "boundary":
res.Accuracy = AccuracyEnum.PostalCodeLevel;
break;
}
}
}
}
else if (search_doc.Search is OsmSharp.Xml.Nominatim.Reverse.v1.reversegeocode)
{
reversegeocode result_v1 = search_doc.Search as OsmSharp.Xml.Nominatim.Reverse.v1.reversegeocode;
if (result_v1.result != null && result_v1.result.Length > 0)
{
double latitude;
double longitude;
if (double.TryParse(result_v1.result[0].lat, System.Globalization.NumberStyles.Float,
System.Globalization.CultureInfo.InvariantCulture, out latitude)
&&
double.TryParse(result_v1.result[0].lon, System.Globalization.NumberStyles.Float,
System.Globalization.CultureInfo.InvariantCulture, out longitude))
{
res.Latitude = latitude;
res.Longitude = longitude;
res.Text = result_v1.result[0].Value;
res.Accuracy = AccuracyEnum.UnkownLocationLevel;
}
}
}
return(res);
}
/// -----------------------------------------------------------------------------
/// <summary>
/// Returns the collection of SearchDocuments populated with Tab MetaData for the given portal.
/// </summary>
/// <param name="portalId"></param>
/// <param name="startDate"></param>
/// <returns></returns>
/// <history>
/// [vnguyen] 04/16/2013 created
/// </history>
/// -----------------------------------------------------------------------------
public override IEnumerable <SearchDocument> GetSearchDocuments(int portalId, DateTime startDate)
{
var searchDocuments = new Dictionary <string, SearchDocument>();
var needReindex = PortalController.GetPortalSettingAsBoolean(UserIndexResetFlag, portalId, false);
if (needReindex)
{
startDate = SqlDateTime.MinValue.Value;
}
try
{
var startUserId = Null.NullInteger;
while (true)
{
var reader = DataProvider.Instance().GetAvailableUsersForIndex(portalId, startDate, startUserId, BatchSize);
int rowsAffected = 0;
var indexedUsers = new List <int>();
while (reader.Read())
{
var userId = Convert.ToInt32(reader["UserId"]);
var displayName = reader["DisplayName"].ToString();
var firstName = reader["FirstName"].ToString();
var propertyName = reader["PropertyName"].ToString();
var propertyValue = reader["PropertyValue"].ToString();
var visibilityMode = ((UserVisibilityMode)Convert.ToInt32(reader["Visibility"]));
var modifiedTime = Convert.ToDateTime(reader["ModifiedTime"]).ToUniversalTime();
//log the userid so that it can get the correct user collection next time.
if (userId > startUserId)
{
startUserId = userId;
}
var uniqueKey = string.Format("{0}_{1}", userId, visibilityMode).ToLowerInvariant();
if (visibilityMode == UserVisibilityMode.FriendsAndGroups)
{
uniqueKey = string.Format("{0}_{1}", uniqueKey, reader["ExtendedVisibility"]);
}
if (searchDocuments.ContainsKey(uniqueKey))
{
var document = searchDocuments[uniqueKey];
document.Body += string.Format(" {0}", propertyValue);
if (modifiedTime > document.ModifiedTimeUtc)
{
document.ModifiedTimeUtc = modifiedTime;
}
}
else
{
//Need remove use exists index for all visibilities.
if (!indexedUsers.Contains(userId))
{
indexedUsers.Add(userId);
}
if (!string.IsNullOrEmpty(propertyValue))
{
var searchDoc = new SearchDocument
{
SearchTypeId = UserSearchTypeId,
UniqueKey = uniqueKey,
PortalId = portalId,
ModifiedTimeUtc = modifiedTime,
Body = propertyValue,
Description = firstName,
Title = displayName
};
searchDocuments.Add(uniqueKey, searchDoc);
}
else if (!searchDocuments.ContainsKey(string.Format("{0}_{1}", userId, UserVisibilityMode.AllUsers).ToLowerInvariant()))
{
if (!indexedUsers.Contains(userId))
{
indexedUsers.Add(userId);
}
//if the user doesn't exist in search collection, we need add it with ALLUsers mode,
//so that can make sure DisplayName will be indexed
var searchDoc = new SearchDocument
{
SearchTypeId = UserSearchTypeId,
UniqueKey = string.Format("{0}_{1}", userId, UserVisibilityMode.AllUsers).ToLowerInvariant(),
PortalId = portalId,
ModifiedTimeUtc = modifiedTime,
Body = string.Empty,
Description = firstName,
Title = displayName
};
searchDocuments.Add(searchDoc.UniqueKey, searchDoc);
}
}
rowsAffected++;
}
//close the data reader
reader.Close();
//remov exists indexes
DeleteDocuments(portalId, indexedUsers);
if (rowsAffected == 0)
{
break;
}
}
if (needReindex)
{
PortalController.DeletePortalSetting(portalId, UserIndexResetFlag);
}
}
catch (Exception ex)
{
Exceptions.Exceptions.LogException(ex);
}
return(searchDocuments.Values);
}
private static SearchDocument CreateSearchDocument(ModuleInfo modInfo, OpenContentSettings settings, JToken contentData, string itemId, string culture, string dataItemTitle, DateTime time)
{
// existance of settings.Template.Main has already been checked: we wouldn't be here if it doesn't exist
// but still, we don't want to count on that too much
var ps = new PortalSettings(modInfo.PortalID);
ps.PortalAlias = PortalAliasController.Instance.GetPortalAlias(ps.DefaultPortalAlias);
string url = null;
// Check if it is a single or list template
if (settings.Template.IsListTemplate)
{
url = TestableGlobals.Instance.NavigateURL(modInfo.TabID, ps, "", $"id={itemId}");
}
else
{
// With a signle template we don't want to identify the content by id.
url = TestableGlobals.Instance.NavigateURL(modInfo.TabID, ps, "");
}
// instanciate the seearch document
var retval = new SearchDocument
{
UniqueKey = modInfo.ModuleID + "-" + itemId + "-" + culture,
PortalId = modInfo.PortalID,
ModifiedTimeUtc = time,
CultureCode = culture,
TabId = modInfo.TabID,
ModuleId = modInfo.ModuleID,
ModuleDefId = modInfo.ModuleDefID,
Url = url
};
// get the title from the template, if it's there
if (!string.IsNullOrEmpty(settings.Template?.Main?.DnnSearchTitle))
{
var dicForHbs = JsonUtils.JsonToDictionary(contentData.ToString());
var hbEngine = new HandlebarsEngine();
retval.Title = hbEngine.ExecuteWithoutFaillure(settings.Template.Main.DnnSearchTitle, dicForHbs, modInfo.ModuleTitle);
}
// SK: this is the behaviour before introduction of DnnSearchTitle
else if (dataItemTitle.IsJson())
{
if (contentData["Title"] != null)
{
retval.Title = contentData["Title"].ToString();
}
else
{
retval.Title = modInfo.ModuleTitle;
}
}
else
{
retval.Title = dataItemTitle;
}
// for the search text, we're using the template in DnnSearchText if it's used
// otherwise, we fall back to previous behaviour:
// - if the item has a field called Description, we use that
// - otherwise just get the whole item contents
if (!string.IsNullOrEmpty(settings.Template?.Main?.DnnSearchText))
{
var dicForHbs = JsonUtils.JsonToDictionary(contentData.ToString());
var hbEngine = new HandlebarsEngine();
retval.Body = hbEngine.ExecuteWithoutFaillure(settings.Template.Main.DnnSearchText, dicForHbs,
modInfo.ModuleTitle);
}
else if (contentData["Description"] != null)
{
retval.Body = contentData["Description"]?.ToString();
}
else
{
retval.Body = JsonToSearchableString(contentData);
}
// for description, we also try and use the available template first
// if that's not there, we'll use the body text for the search document
if (!string.IsNullOrEmpty(settings.Template?.Main?.DnnSearchDescription))
{
var dicForHbs = JsonUtils.JsonToDictionary(contentData.ToString());
var hbEngine = new HandlebarsEngine();
retval.Description = hbEngine.ExecuteWithoutFaillure(settings.Template.Main.DnnSearchDescription, dicForHbs,
modInfo.ModuleTitle);
}
else
{
retval.Description = retval.Body;
}
retval.Title = HttpUtility.HtmlDecode(retval.Title).StripHtml();
retval.Body = HttpUtility.HtmlDecode(retval.Body).StripHtml();
retval.Description = HttpUtility.HtmlDecode(retval.Description).StripHtml();
// Add support for module terms
if (modInfo.Terms != null && modInfo.Terms.Count > 0)
{
retval.Tags = CollectHierarchicalTags(modInfo.Terms);
}
return(retval);
}
public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo modInfo, DateTime beginDateUtc)
{
Log.Logger.TraceFormat("Indexing content Module {0} - Tab {1} - indexing from {3}", modInfo.ModuleID, modInfo.TabID, modInfo.CultureCode, beginDateUtc);
var searchDocuments = new List <SearchDocument>();
//If module is marked as "don't index" then return no results
if (modInfo.ModuleSettings.GetValue("AllowIndex", "True") == "False")
{
Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - MODULE Indexing disabled", modInfo.ModuleID, modInfo.CultureCode);
return(searchDocuments);
}
//If tab of the module is marked as "don't index" then return no results
if (modInfo.ParentTab.TabSettings.GetValue("AllowIndex", "True") == "False")
{
Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - TAB Indexing disabled", modInfo.ModuleID, modInfo.CultureCode);
return(searchDocuments);
}
//If tab is marked as "inactive" then return no results
if (modInfo.ParentTab.DisableLink)
{
Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - TAB is inactive", modInfo.ModuleID, modInfo.CultureCode);
return(searchDocuments);
}
var module = new OpenContentModuleInfo(modInfo);
OpenContentSettings settings = modInfo.OpenContentSettings();
//if (settings.Template?.Main == null || !settings.Template.Main.DnnSearch)
//{
// Log.Logger.TraceFormat("Indexing content Module {0} - Tab {1} - settings.Template?.Main == null || !settings.Template.Main.DnnSearch", modInfo.ModuleID, modInfo.TabID, modInfo.CultureCode);
// return searchDocuments;
//}
if (settings.IsOtherModule)
{
return(searchDocuments);
}
IDataSource ds = DataSourceManager.GetDataSource(module.Settings.Manifest.DataSource);
var dsContext = OpenContentUtils.CreateDataContext(module);
IDataItems contentList = ds.GetAll(dsContext, null);
if (!contentList.Items.Any())
{
Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - No content found", modInfo.ModuleID, modInfo.CultureCode);
}
foreach (IDataItem content in contentList.Items)
{
if (content == null)
{
Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - Content is Null", modInfo.ModuleID, modInfo.CultureCode);
}
else if (content.LastModifiedOnDate.ToUniversalTime() > beginDateUtc &&
content.LastModifiedOnDate.ToUniversalTime() < DateTime.UtcNow)
{
SearchDocument searchDoc;
if (DnnLanguageUtils.IsMultiLingualPortal(modInfo.PortalID))
{
searchDoc = GetLocalizedItem(modInfo, settings, content);
searchDocuments.Add(searchDoc);
if (modInfo.LocalizedModules != null)
{
foreach (var localizedModule in modInfo.LocalizedModules)
{
SearchDocument localizedSearchDoc = GetLocalizedItem(localizedModule.Value, settings, content);
searchDocuments.Add(localizedSearchDoc);
}
}
}
else
{
searchDoc = CreateSearchDocument(modInfo, settings, content.Data, content.Id, "", content.Title, JsonToSearchableString(content.Data), content.LastModifiedOnDate.ToUniversalTime());
searchDocuments.Add(searchDoc);
Log.Logger.TraceFormat("Indexing content {0}|{5} - OK! {1} ({2}) of {3}", modInfo.ModuleID, searchDoc.Title, modInfo.TabID, content.LastModifiedOnDate.ToUniversalTime(), modInfo.CultureCode);
}
}
else
{
Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - No need to index: lastmod {2} ", modInfo.ModuleID, modInfo.CultureCode, content.LastModifiedOnDate.ToUniversalTime());
}
}
return(searchDocuments);
}
/// <inheritdoc />
public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo moduleInfo, DateTime beginDateUtc)
{
if (moduleInfo == null)
{
throw new ArgumentNullException("modInfo", @"modInfo must not be null.");
}
var searchItems = new List <SearchDocument>();
// only index the JobDetail module definition (since most of this information is only viewable there,
// and because the Guid parameter on the SearchItemInfo ("jobid=" + jobid) gets put on the querystring to make it work all automagically). BD
if (ModuleDefinitionController.GetModuleDefinitionByFriendlyName(
ModuleDefinition.JobDetail.ToString(),
moduleInfo.DesktopModuleID)
.ModuleDefID
!= moduleInfo.ModuleDefID ||
!ModuleSettings.JobDetailEnableDnnSearch.GetValueAsBooleanFor(
DesktopModuleName,
moduleInfo,
ModuleSettings.JobDetailEnableDnnSearch.DefaultValue))
{
return(searchItems);
}
int?jobGroupId = ModuleSettings.JobGroupId.GetValueAsInt32For(
DesktopModuleName,
moduleInfo,
ModuleSettings.JobGroupId.DefaultValue);
// TODO: only retrieve jobs modified after beginDateUtc
using (IDataReader jobs = DataProvider.Instance()
.GetJobs(jobGroupId, moduleInfo.PortalID))
{
while (jobs.Read())
{
var jobId = ((int)jobs["JobId"]).ToString(CultureInfo.InvariantCulture);
var revisionDate = (DateTime)jobs["RevisionDate"];
var searchDocument = new SearchDocument
{
UniqueKey = "Employment_Job_" + jobId,
ModifiedTimeUtc = revisionDate.ToUniversalTime(),
ModuleId = moduleInfo.ModuleID,
PortalId = moduleInfo.PortalID,
QueryString = "jobid=" + jobId,
};
searchItems.Add(searchDocument);
if ((bool)jobs["IsFilled"])
{
searchDocument.IsActive = false;
continue;
}
var searchDescription = HtmlUtils.StripWhiteSpace(
HtmlUtils.Clean((string)jobs["JobDescription"], false),
true);
var searchItemTitle = string.Format(
CultureInfo.CurrentCulture,
Utility.GetString("JobInLocation", LocalResourceFile, moduleInfo.PortalID),
(string)jobs["JobTitle"],
(string)jobs["LocationName"],
(string)jobs["StateName"]);
var searchedContent = HtmlUtils.StripWhiteSpace(
HtmlUtils.Clean(
(string)jobs["JobTitle"]
+ " "
+ (string)jobs["JobDescription"]
+ " "
+ (string)jobs["RequiredQualifications"]
+ " "
+ (string)jobs["DesiredQualifications"],
false),
true);
searchDocument.Title = searchItemTitle;
searchDocument.Description = searchDescription;
searchDocument.AuthorUserId = (int)jobs["RevisingUser"];
searchDocument.Body = searchedContent;
}
}
return(searchItems);
}
public static bool IsVisible(this SearchDocument doc, Guid?userIdenfitier)
{
return(null != doc && (null == doc.UserIdentifiers || doc.UserIdentifiers.Any(id => id == userIdenfitier)));
}
public void AddSearchDocument(SearchDocument searchDocument)
{
this.AddSearchDocumentInternal(searchDocument, false);
}
public static Document ToDocument(this SearchDocument content)
{
var doc = new Document();
doc.Add(new Field(SearchDocument.IdentifierKey, content.Identifier.ToString(), Field.Store.YES, Field.Index.NO));
doc.Add(new Field(SearchDocument.UserIdentifierKey, content.UserIdentifier.ToString(), Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field(SearchDocument.MemberNameKey, content.MemberName, Field.Store.YES, Field.Index.ANALYZED));
var typeField = new NumericField(SearchDocument.TypeKey, Field.Store.YES, true);
typeField.SetIntValue((int)content.Type);
doc.Add(typeField);
var latField = new NumericField(SearchDocument.LatitudeKey, Field.Store.YES, true);
latField.SetDoubleValue(content.Latitude);
doc.Add(latField);
var longField = new NumericField(SearchDocument.LongitudeKey, Field.Store.YES, true);
longField.SetDoubleValue(content.Longitude);
doc.Add(longField);
doc.Add(new Field(SearchDocument.CreatedOnKey, content.CreatedOn.ToStringExact(), Field.Store.YES, Field.Index.NO));
var permissions = content.Permissions();
if (!string.IsNullOrWhiteSpace(permissions))
{
doc.Add(new Field(SearchDocument.PermissionsKey, permissions, Field.Store.YES, Field.Index.NO));
}
if (!string.IsNullOrWhiteSpace(content.ImageData))
{
doc.Add(new Field(SearchDocument.ImageDataKey, content.ImageData, Field.Store.YES, Field.Index.NO));
}
if (!string.IsNullOrWhiteSpace(content.Title))
{
doc.Add(new Field(SearchDocument.TitleKey, content.Title, Field.Store.YES, Field.Index.NO));
}
if (!string.IsNullOrWhiteSpace(content.Description))
{
doc.Add(new Field(SearchDocument.DescriptionKey, content.Description, Field.Store.YES, Field.Index.NO));
}
if (!string.IsNullOrWhiteSpace(content.Key))
{
doc.Add(new Field(SearchDocument.KeyKey, content.Key, Field.Store.YES, Field.Index.NOT_ANALYZED));
}
if (!string.IsNullOrWhiteSpace(content.Content))
{
doc.Add(new Field(SearchDocument.ContentKey, content.Content, Field.Store.YES, Field.Index.ANALYZED));
}
if (!string.IsNullOrWhiteSpace(content.Location))
{
doc.Add(new Field(SearchDocument.LocationKey, content.Location, Field.Store.YES, Field.Index.ANALYZED));
}
return(doc);
}
/// -----------------------------------------------------------------------------
/// <summary>
/// Returns the collection of SearchDocuments populated with Tab MetaData for the given portal.
/// </summary>
/// <param name="portalId"></param>
/// <param name="startDateLocal"></param>
/// <returns></returns>
/// <history>
/// [vnguyen] 04/16/2013 created
/// </history>
/// -----------------------------------------------------------------------------
public override IEnumerable <SearchDocument> GetSearchDocuments(int portalId, DateTime startDateLocal)
{
var searchDocuments = new Dictionary <string, SearchDocument>();
var needReindex = PortalController.GetPortalSettingAsBoolean(UserIndexResetFlag, portalId, false);
if (needReindex)
{
startDateLocal = SqlDateTime.MinValue.Value.AddDays(1);
}
var controller = new ListController();
var textDataType = controller.GetListEntryInfo("DataType", "Text");
var richTextDataType = controller.GetListEntryInfo("DataType", "RichText");
var profileDefinitions = ProfileController.GetPropertyDefinitionsByPortal(portalId, false, false)
.Cast <ProfilePropertyDefinition>()
.Where(d => (textDataType != null && d.DataType == textDataType.EntryID) ||
(richTextDataType != null && d.DataType == richTextDataType.EntryID)).ToList();
try
{
var startUserId = Null.NullInteger;
while (true)
{
var reader = DataProvider.Instance()
.GetAvailableUsersForIndex(portalId, startDateLocal, startUserId, BatchSize);
int rowsAffected = 0;
var indexedUsers = new List <int>();
while (reader.Read())
{
var userSearch = GetUserSearch(reader);
AddBasicInformation(searchDocuments, indexedUsers, userSearch, portalId);
//log the userid so that it can get the correct user collection next time.
if (userSearch.UserId > startUserId)
{
startUserId = userSearch.UserId;
}
foreach (var definition in profileDefinitions)
{
var propertyName = definition.PropertyName;
if (!ContainsColumn(propertyName, reader))
{
continue;
}
var propertyValue = reader[propertyName].ToString();
if (string.IsNullOrEmpty(propertyValue) || !propertyValue.Contains(ValueSplitFlag))
{
continue;
}
var splitValues = Regex.Split(propertyValue, Regex.Escape(ValueSplitFlag));
propertyValue = splitValues[0];
var visibilityMode = ((UserVisibilityMode)Convert.ToInt32(splitValues[1]));
var extendedVisibility = splitValues[2];
var modifiedTime = Convert.ToDateTime(splitValues[3]).ToUniversalTime();
if (string.IsNullOrEmpty(propertyValue))
{
continue;
}
var uniqueKey = string.Format("{0}_{1}", userSearch.UserId, visibilityMode).ToLowerInvariant();
if (visibilityMode == UserVisibilityMode.FriendsAndGroups)
{
uniqueKey = string.Format("{0}_{1}", uniqueKey, extendedVisibility);
}
if (searchDocuments.ContainsKey(uniqueKey))
{
var document = searchDocuments[uniqueKey];
document.Keywords.Add(propertyName, propertyValue);
if (modifiedTime > document.ModifiedTimeUtc)
{
document.ModifiedTimeUtc = modifiedTime;
}
}
else
{
//Need remove use exists index for all visibilities.
if (!indexedUsers.Contains(userSearch.UserId))
{
indexedUsers.Add(userSearch.UserId);
}
if (!string.IsNullOrEmpty(propertyValue))
{
var searchDoc = new SearchDocument
{
SearchTypeId = UserSearchTypeId,
UniqueKey = uniqueKey,
PortalId = portalId,
ModifiedTimeUtc = modifiedTime,
Description = userSearch.FirstName,
Title = userSearch.DisplayName
};
searchDoc.Keywords.Add(propertyName, propertyValue);
searchDoc.NumericKeys.Add("superuser", Convert.ToInt32(userSearch.SuperUser));
searchDocuments.Add(uniqueKey, searchDoc);
}
}
}
rowsAffected++;
}
//close the data reader
reader.Close();
//remov exists indexes
DeleteDocuments(portalId, indexedUsers);
if (rowsAffected == 0)
{
break;
}
}
if (needReindex)
{
PortalController.DeletePortalSetting(portalId, UserIndexResetFlag);
}
}
catch (Exception ex)
{
Exceptions.Exceptions.LogException(ex);
}
return(searchDocuments.Values);
}
private ISearchResults Search <T>(ISearchCriteria criteria) where T : EntryContentBase
{
// set up filters
// 1: phrase
var filters = new List <Nest.QueryContainer>();
if (criteria is CatalogEntrySearchCriteria)
{
var cesc = criteria as CatalogEntrySearchCriteria;
if (!string.IsNullOrWhiteSpace(cesc.SearchPhrase))
{
filters.Add(new Nest.QueryContainerDescriptor <T>().SimpleQueryString(
sq => sq.Fields(f => f.Field("*.analyzed")).Query(cesc.SearchPhrase.Trim())));
}
}
// 2: id ... NOTE: Vulcan supports 1 and only 1 filter field, code
if (criteria.ActiveFilterFields != null && criteria.ActiveFilterFields.Count() == 1 && criteria.ActiveFilterFields[0].Equals("code", StringComparison.InvariantCultureIgnoreCase))
{
filters.Add(new Nest.QueryContainerDescriptor <T>().Term(
p => p.Field(f => f.Code).Value((criteria.ActiveFilterValues[0] as SimpleValue).value)));
}
// 3: inactive... TODO, not sure what this should check!
/*
* if(!criteria.IncludeInactive)
* {
* filters.Add(new Nest.QueryContainerDescriptor<T>().Term(
* p => p.Field(f => f.)
* }*/
// get catalog filter, if needed
var catalogReferences = new List <ContentReference>();
if (criteria is CatalogEntrySearchCriteria)
{
var cesc = criteria as CatalogEntrySearchCriteria;
if (cesc.CatalogNames != null)
{
var catalogs = ContentLoader.Service.GetChildren <CatalogContent>(ReferenceConverter.Service.GetRootLink());
if (catalogs != null && catalogs.Any())
{
foreach (var catalogName in cesc.CatalogNames)
{
var catalog = catalogs.FirstOrDefault(c => c.Name.Equals(catalogName, StringComparison.InvariantCultureIgnoreCase));
if (catalog != null)
{
catalogReferences.Add(catalog.ContentLink);
}
}
}
}
}
if (!catalogReferences.Any())
{
catalogReferences = null;
}
// do search
var searchDescriptor = new Nest.SearchDescriptor <T>();
searchDescriptor.Skip(criteria.StartingRecord);
searchDescriptor.Take(criteria.RecordsToRetrieve);
if (filters.Any())
{
searchDescriptor.Query(q => q.Bool(b => b.Must(filters.ToArray())));
}
var client = VulcanHandler.Service.GetClient(new CultureInfo(criteria.Locale));
var results = client.SearchContent <T>(q => searchDescriptor, false, catalogReferences);
//var id = ReferenceConverter.Service.GetObjectId();
var searchDocuments = new SearchDocuments()
{
TotalCount = Convert.ToInt32(results.Total)
};
if (results.Hits != null && results.Hits.Any())
{
foreach (var hit in results.Hits)
{
var doc = new SearchDocument();
doc.Add(new SearchField("_id", ReferenceConverter.Service.GetObjectId(hit.Source.ContentLink)));
searchDocuments.Add(doc);
}
}
return(new SearchResults(searchDocuments, criteria));
}
public bool ReadDocuments(string dataFilePath)
{
List<SearchDocument> tmpDocuments = new List<SearchDocument>();
StringBuilder sb = new StringBuilder();
string header = string.Empty;
try
{
using (FileStream fs = new FileStream(dataFilePath, FileMode.Open, FileAccess.Read))
{
using (StreamReader sr = new StreamReader(fs))
{
while (!sr.EndOfStream)
{
sb.Remove(0, sb.Length);
string line;
int counter = 0;
header = string.Empty;
while(((line = sr.ReadLine()) != string.Empty))
{
if (counter == 0)
header = string.Format("{0} ", line);
else
sb.Append(string.Format("{0} ", line));
counter++;
if (sr.EndOfStream)
break;
}
SearchDocument document = new SearchDocument(header, sb.ToString());
tmpDocuments.Add(document);
}
}
}
}
catch (FileNotFoundException ex)
{
throw new ArgumentException(String.Format("Plik nie został znaleziony. {0}", ex.Message));
}
catch (IOException ex)
{
throw new ArgumentException(String.Format("Błąd podczas odczytu pliku", ex.Message));
}
// czyszczenie i stemming wczytanych dokumentów
if (tmpDocuments.Count > 0)
{
for(int i=0; i < tmpDocuments.Count; i++)
tmpDocuments[i].ProcessElement(textProcessor);
documents = tmpDocuments;
return true;
}
else
return false;
}
public void ModifyingDocumentsDoesNotCreateDuplicates()
{
// Arrange
const string tabUrl = "mysite/ContentUrl";
const string title = "content title";
const string contentBody = "content body";
const string titleModified = title + " modified";
var uniqueKey = Guid.NewGuid().ToString();
var now = DateTime.UtcNow;
var originalDocument = new SearchDocument
{
UniqueKey = uniqueKey,
TabId = TabId1,
Url = tabUrl,
Title = title,
Body = contentBody,
SearchTypeId = TabSearchTypeId,
ModifiedTimeUtc = now,
PortalId = PortalId0,
RoleId = RoleId731,
Keywords = { { "description", "mycontent" } },
NumericKeys = { { "points", 5 } },
};
this.internalSearchController.AddSearchDocument(originalDocument);
this.internalSearchController.Commit();
var modifiedDocument = new SearchDocument
{
UniqueKey = uniqueKey,
TabId = TabId1,
Url = tabUrl,
Title = titleModified,
Body = contentBody + " modified",
SearchTypeId = TabSearchTypeId,
ModifiedTimeUtc = now,
PortalId = PortalId0,
RoleId = RoleId731,
Keywords = { { "description", "mycontent_modified" }, { "description2", "mycontent_modified" } },
NumericKeys = { { "points", 8 }, { "point2", 7 } },
};
this.internalSearchController.AddSearchDocument(modifiedDocument);
this.internalSearchController.Commit();
var query = new SearchQuery
{
KeyWords = title,
PortalIds = new List <int> {
PortalId0
},
SearchTypeIds = new[] { ModuleSearchTypeId, TabSearchTypeId, UserSearchTypeId },
BeginModifiedTimeUtc = now.AddMinutes(-1),
EndModifiedTimeUtc = now.AddMinutes(+1),
PageIndex = 1,
PageSize = 15,
SortField = 0,
TitleSnippetLength = 120,
BodySnippetLength = 300,
WildCardSearch = true,
};
// Run
var searchResults = this.GetGroupedDetailViewResults(query).ToList();
// Assert
Assert.AreEqual(1, searchResults.Count());
Assert.AreEqual(1, searchResults.First().Results.Count);
Assert.AreEqual(tabUrl, searchResults.First().Results.First().DocumentUrl);
Assert.AreEqual(titleModified, searchResults.First().Results.First().Title);
}
/// -----------------------------------------------------------------------------
/// <summary>
/// Implements the search interface for DotNetNuke
/// </summary>
/// -----------------------------------------------------------------------------
public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo modInfo, DateTime beginDateUtc)
{
var searchDocuments = new List <SearchDocument>();
var udtController = new UserDefinedTableController(modInfo);
try
{
var dsUserDefinedRows = udtController.GetDataSet(withPreRenderedValues: false);
//Get names of ChangedBy and ChangedAt columns
var colnameChangedBy = udtController.ColumnNameByDataType(dsUserDefinedRows,
DataTypeNames.UDT_DataType_ChangedBy);
var colnameChangedAt = udtController.ColumnNameByDataType(dsUserDefinedRows,
DataTypeNames.UDT_DataType_ChangedAt);
var settings = modInfo.ModuleSettings;
var includeInSearch = !(settings[SettingName.ExcludeFromSearch].AsBoolean());
if (includeInSearch)
{
foreach (DataRow row in dsUserDefinedRows.Tables[DataSetTableName.Data].Rows)
{
var changedDate = DateTime.Today;
var changedByUserId = 0;
if (colnameChangedAt != string.Empty && !Information.IsDBNull(row[colnameChangedAt]))
{
changedDate = Convert.ToDateTime(row[colnameChangedAt]);
}
if (colnameChangedBy != string.Empty && !Information.IsDBNull(row[colnameChangedBy]))
{
changedByUserId = ModuleSecurity.UserId(row[colnameChangedBy].ToString(), modInfo.PortalID);
}
var desc = string.Empty;
foreach (DataRow col in dsUserDefinedRows.Tables[DataSetTableName.Fields].Rows)
{
var fieldType = col[FieldsTableColumn.Type].ToString();
var fieldTitle = col[FieldsTableColumn.Title].ToString();
var visible = Convert.ToBoolean(col[FieldsTableColumn.Visible]);
if (visible &&
(fieldType.StartsWith("Text") || fieldType == DataTypeNames.UDT_DataType_String))
{
desc += string.Format("{0} • ", Convert.ToString(row[fieldTitle]));
}
}
if (desc.EndsWith("<br/>"))
{
desc = desc.Substring(0, Convert.ToInt32(desc.Length - 5));
}
var searchDoc = new SearchDocument
{
UniqueKey = row[DataTableColumn.RowId].ToString(),
PortalId = modInfo.PortalID,
Title = modInfo.ModuleTitle,
Description = desc,
Body = desc,
ModifiedTimeUtc = changedDate
};
searchDocuments.Add(searchDoc);
}
}
}
catch (Exception ex)
{
Exceptions.LogException(ex);
}
return(searchDocuments);
}
public virtual void IncludeDefaultField(Document document, IContent content)
{
var versionable = (content as IVersionable);
var changeTrackable = content as IChangeTrackable;
var contentStatus = ((int)versionable.Status).ToString();
var virtualPath = ContentIndexHelpers.GetContentVirtualpath(content);
var acl = ContentIndexHelpers.GetContentACL(content);
var contentType = ContentIndexHelpers.GetContentType(content);
var language = (content as ILocalizable).Language.Name;
var startPublishDate = versionable.StartPublish.HasValue ? DateTools.DateToString(versionable.StartPublish.Value.ToUniversalTime(), DateTools.Resolution.SECOND) : string.Empty;
var stopPublishDate = versionable.StopPublish.HasValue ? DateTools.DateToString(versionable.StopPublish.Value.ToUniversalTime(), DateTools.Resolution.SECOND) : string.Empty;
var expired = "";
if ((versionable.StopPublish.HasValue && versionable.StopPublish.Value.ToUniversalTime() < DateTime.UtcNow) ||
(versionable.StartPublish.HasValue && versionable.StartPublish.Value.ToUniversalTime() > DateTime.UtcNow))
{
expired = "true";
}
var createdDate = DateTools.DateToString(changeTrackable.Created.ToUniversalTime(), DateTools.Resolution.SECOND);
var updatedDate = DateTools.DateToString(changeTrackable.Changed.ToUniversalTime(), DateTools.Resolution.SECOND);
var idField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_ID),
SearchDocument.FormatDocumentId(content), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var nameField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_NAME),
content.Name, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var statusField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_STATUS),
contentStatus, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var virtualPathField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_VIRTUAL_PATH),
virtualPath, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var aclField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_ACL),
acl, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var typeField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_TYPE),
contentType, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var langField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_LANGUAGE),
language, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var startPublishField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_START_PUBLISHDATE),
startPublishDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var stopPublishField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_STOP_PUBLISHDATE),
stopPublishDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var referenceField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_REFERENCE),
content.ContentLink.ID.ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var createdField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_CREATED),
createdDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var updatedField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_CHANGED),
updatedDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
var expiredField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_EXPIRED),
expired, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES);
document.Add(idField);
document.Add(nameField);
document.Add(statusField);
document.Add(virtualPathField);
document.Add(aclField);
document.Add(typeField);
document.Add(langField);
document.Add(startPublishField);
document.Add(stopPublishField);
document.Add(referenceField);
document.Add(createdField);
document.Add(updatedField);
document.Add(expiredField);
}
public async Task <IActionResult> GetSearchDocument([FromQuery] SearchDocument SearchDocument) //int catId, int subcatId, int officeId, int companyId, string date)
{
//var items = await _context.Document.Where(x => EF.Functions.FreeText(x.DescrizioneDocumento, SearchDocument.SearchText)).ToListAsync();
if (SearchDocument.SearchText != null)
{
SearchDocument.SearchText = SearchDocument.SearchText.TrimEnd();
SearchDocument.SearchText = string.Join(" ", SearchDocument.SearchText.Split(' ').Select(w => string.Format("+{0}*", w)));
}
if (!ModelState.IsValid)
{
return(BadRequest(ModelState));
}
SearchDocument.EndDate = SearchDocument.EndDate.Date.AddHours(23).AddMinutes(59).AddSeconds(59);
List <Document> document;
int totalCount;
if (SearchDocument.SearchText != null)
{
document = await _context.Document.FromSql("CALL FullTextSearch({0})", SearchDocument.SearchText).ToListAsync();
}
else
{
document = await _context.Document.ToListAsync();
}
if (SearchDocument.CategoriaId != 0 || SearchDocument.SottocategoriaId != 0 || SearchDocument.SocietaId != 0 || SearchDocument.UfficioId != 0 || SearchDocument.StartDate != new DateTime(0001, 1, 1) || SearchDocument.Parte2Id != 0 || SearchDocument.Parte3Id != 0)
{
if (SearchDocument.CategoriaId != 0)
{
document = document.Where(x => x.CategoriaId == SearchDocument.CategoriaId).ToList();
}
if (SearchDocument.SottocategoriaId != 0)
{
document = document.Where(x => x.SottocategoriaId == SearchDocument.SottocategoriaId).ToList();
}
if (SearchDocument.SocietaId != 0)
{
document = document.Where(x => x.SocietaId == SearchDocument.SocietaId).ToList();
}
if (SearchDocument.UfficioId != 0)
{
document = document.Where(x => x.UfficioId == SearchDocument.UfficioId).ToList();
}
if (SearchDocument.Parte2Id != 0)
{
document = document.Where(x => x.Parte2Id == SearchDocument.Parte2Id).ToList();
}
if (SearchDocument.Parte3Id != 0)
{
document = document.Where(x => x.Parte3Id == SearchDocument.Parte3Id).ToList();
}
if (SearchDocument.StartDate != new DateTime(0001, 1, 1))
{
document = document.Where(x => x.DataDocumento >= SearchDocument.StartDate.Date && x.DataDocumento <= SearchDocument.EndDate).ToList();
}
}
int p = (SearchDocument.CurrentPage * 20) - 20;
totalCount = document.Count();
if (SearchDocument.SortColumn != null && SearchDocument.SortDirection != null)
{
switch (SearchDocument.SortColumn)
{
case "dataScadenza":
if (SearchDocument.SortDirection == "asc")
{
document = document.OrderBy(x => x.DataScadenza).Skip(p).Take(20).ToList();
}
else
{
document = document.OrderByDescending(x => x.DataScadenza).Skip(p).Take(20).ToList();
}
break;
case "dataDocumento":
if (SearchDocument.SortDirection == "asc")
{
document = document.OrderBy(x => x.DataDocumento).Skip(p).Take(20).ToList();
}
else
{
document = document.OrderByDescending(x => x.DataDocumento).Skip(p).Take(20).ToList();
}
break;
default:
document = document.OrderByDescending(x => x.DataDocumento).Skip(p).Take(20).ToList();
break;
}
}
else
{
document = document.OrderByDescending(x => x.DataDocumento).Skip(p).Take(20).ToList();
}
Response.Headers.Add("X-Pagination", JsonConvert.SerializeObject(totalCount));
return(Ok(document));
}
public Result GetResults(Query searchQuery)
{
Uri searchEndpoint = new Uri(_configuration["SearchServiceEndpoint"]);
string apiKey = _configuration["SearchServiceQueryApiKey"];
string indexName = _configuration["SearchIndexName"];
Result results = new Result();
List <Dictionary <string, string> > processedResults = new List <Dictionary <string, string> >();
Dictionary <string, List <FacetResult> > facetResults = new Dictionary <string, List <FacetResult> >();
Dictionary <string, string> speakersDict = _configuration.GetSection("Speakers").GetChildren().ToDictionary(x => x.Key, x => x.Value);
SearchOptions options = new SearchOptions
{
Size = 100,
IncludeTotalCount = true,
Facets = { "duration,sort:count", "recognizedPhrases/speaker,sort:count" },
HighlightFields = { "combinedRecognizedPhrases/display" },
HighlightPreTag = "<span class='hitHighlight'>",
HighlightPostTag = "</span>",
Filter = searchQuery.filter,
SearchMode = SearchMode.All
};
AzureKeyCredential credential = new AzureKeyCredential(apiKey);
SearchClient client = new SearchClient(searchEndpoint, indexName, credential);
SearchResults <SearchDocument> response = client.Search <SearchDocument>(searchQuery.searchText, options);
foreach (SearchResult <SearchDocument> result in response.GetResults())
{
SearchDocument doc = result.Document;
Dictionary <string, string> r = new Dictionary <string, string>();
string speakerTranscription = string.Empty;
r.Add("filename", (string)doc["metadata_storage_name"]);
r.Add("timestamp", (string)doc["timeStamp"]);
//results.processedResults.Add("size", (string)doc["metadata_storage_size"]);
r.Add("lastmodified", (string)doc["metadata_storage_last_modified"]);
r.Add("transcription", (string)doc.GetObjectCollection("combinedRecognizedPhrases")[0]["display"]);
foreach (SearchDocument recognizedPhrase in doc.GetObjectCollection("recognizedPhrases"))
{
speakerTranscription += String.Format("<span class=\"speakerLabel\">Speaker {0}</span>: {1} <br>", recognizedPhrase.GetInt32("speaker").ToString(), recognizedPhrase.GetObjectCollection("nBest")[0].GetString("display"));
}
r.Add("speakerTranscription", speakerTranscription);
if (result.Highlights != null)
{
r.Add("highlights", String.Join(" ... ", result.Highlights["combinedRecognizedPhrases/display"].ToList()));
}
else
{
r.Add("highlights", r["transcription"].Substring(0, Math.Min(r["transcription"].Length, 200)) + "...");
}
r.Add("source", (string)doc["source"]);
processedResults.Add(r);
}
foreach (var f in response.Facets)
{
switch (f.Key)
{
case "recognizedPhrases/speaker":
facetResults["recognizedPhrases/speaker"] = f.Value.ToList();
continue;
case "duration":
facetResults["duration"] = f.Value.ToList();
continue;
}
}
results.processedResults = processedResults;
results.facets = facetResults;
results.count = response.TotalCount;
return(results);
}
public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo moduleInfo, DateTime beginDate)
{
var searchDocuments = new Dictionary <string, SearchDocument>();
int lastJournalId = Null.NullInteger;
if (beginDate == DateTime.MinValue)
{
beginDate = SqlDateTime.MinValue.Value;
}
if (beginDate > SqlDateTime.MinValue.Value)
{
beginDate = beginDate.ToUniversalTime();
}
try
{
while (true)
{
var reader = DataProvider.Instance()
.ExecuteReader("Journal_GetSearchItems", moduleInfo.PortalID,
moduleInfo.TabModuleID, beginDate, lastJournalId,
Constants.SearchBatchSize);
var journalIds = new Dictionary <int, int>();
while (reader.Read())
{
var journalId = Convert.ToInt32(reader["JournalId"]);
var journalTypeId = reader["JournalTypeId"].ToString();
var userId = Convert.ToInt32(reader["UserId"]);
var dateUpdated = Convert.ToDateTime(reader["DateUpdated"]);
var profileId = reader["ProfileId"].ToString();
var groupId = reader["GroupId"].ToString();
var title = reader["Title"].ToString();
var summary = reader["Summary"].ToString();
var securityKey = reader["SecurityKey"].ToString();
var tabId = reader["TabId"].ToString();
var tabModuleId = reader["ModuleId"].ToString();
var key = string.Format("JI_{0}", journalId);
if (searchDocuments.ContainsKey(key))
{
searchDocuments[key].UniqueKey +=
string.Format(",{0}", securityKey);
}
else
{
var searchDocument = new SearchDocument()
{
UniqueKey = string.Format("JI_{0}_{1}", journalId, securityKey),
Body = summary,
ModifiedTimeUtc = dateUpdated,
Title = title,
AuthorUserId = userId,
Keywords = new Dictionary <string, string>()
{
{ "TabId", tabId },
{ "TabModuleId", tabModuleId },
{ "ProfileId", profileId },
{ "GroupId", groupId }
}
};
searchDocuments.Add(key, searchDocument);
}
if (journalId > lastJournalId)
{
lastJournalId = journalId;
}
if (!journalIds.ContainsKey(journalId))
{
journalIds.Add(journalId, userId);
}
}
//close the reader
reader.Close();
if (journalIds.Count == 0)
{
break;
}
else
{
//index comments for this journal
AddCommentItems(journalIds, searchDocuments);
}
}
}
catch (Exception ex)
{
Exceptions.LogException(ex);
}
return(searchDocuments.Values.ToList());
}
private static int FindModifiedUsers(int portalId, DateTime startDateLocal,
IDictionary <string, SearchDocument> searchDocuments, IList <ProfilePropertyDefinition> profileDefinitions, out IList <int> indexedUsers,
ref int startUserId)
{
var rowsAffected = 0;
indexedUsers = new List <int>();
using (var reader = DataProvider.Instance().GetAvailableUsersForIndex(portalId, startDateLocal, startUserId, BatchSize))
{
while (reader.Read())
{
var userSearch = GetUserSearch(reader);
if (userSearch == null)
{
continue;
}
AddBasicInformation(searchDocuments, indexedUsers, userSearch, portalId);
//log the userid so that it can get the correct user collection next time.
if (userSearch.UserId > startUserId)
{
startUserId = userSearch.UserId;
}
foreach (var definition in profileDefinitions)
{
var propertyName = definition.PropertyName;
if (!ContainsColumn(propertyName, reader))
{
continue;
}
var propertyValue = reader[propertyName].ToString();
if (string.IsNullOrEmpty(propertyValue) || !propertyValue.Contains(ValueSplitFlag))
{
continue;
}
var splitValues = Regex.Split(propertyValue, Regex.Escape(ValueSplitFlag));
propertyValue = splitValues[0];
var visibilityMode = ((UserVisibilityMode)Convert.ToInt32(splitValues[1]));
var extendedVisibility = splitValues[2];
var modifiedTime = Convert.ToDateTime(splitValues[3]).ToUniversalTime();
if (string.IsNullOrEmpty(propertyValue))
{
continue;
}
//DNN-5740: replace split flag if it included in property value.
propertyValue = propertyValue.Replace("[$][$][$]", "$$$");
var uniqueKey = string.Format("{0}_{1}", userSearch.UserId, visibilityMode).ToLowerInvariant();
if (visibilityMode == UserVisibilityMode.FriendsAndGroups)
{
uniqueKey = string.Format("{0}_{1}", uniqueKey, extendedVisibility);
}
if (searchDocuments.ContainsKey(uniqueKey))
{
var document = searchDocuments[uniqueKey];
document.Keywords.Add(propertyName, propertyValue);
if (modifiedTime > document.ModifiedTimeUtc)
{
document.ModifiedTimeUtc = modifiedTime;
}
}
else
{
//Need remove use exists index for all visibilities.
if (!indexedUsers.Contains(userSearch.UserId))
{
indexedUsers.Add(userSearch.UserId);
}
if (!string.IsNullOrEmpty(propertyValue))
{
var searchDoc = new SearchDocument
{
SearchTypeId = UserSearchTypeId,
UniqueKey = uniqueKey,
PortalId = portalId,
ModifiedTimeUtc = modifiedTime,
Description = userSearch.FirstName,
Title = userSearch.DisplayName
};
searchDoc.Keywords.Add(propertyName, propertyValue);
searchDoc.NumericKeys.Add("superuser", Convert.ToInt32(userSearch.SuperUser));
searchDocuments.Add(uniqueKey, searchDoc);
}
}
}
rowsAffected++;
}
}
return(rowsAffected);
}
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddImageUrl()
{
var entry = GetCatalogEntryRow("Product");
_urlResolverMock.Setup(x => x.GetUrl(new ContentReference(5, 0)))
.Returns("http://myimage");
_assetUrlConventionsMock.Setup(x => x.GetDefaultGroup(It.IsAny<IAssetContainer>()))
.Returns("default");
var document = new SearchDocument();
_subject.UpdateSearchDocument(ref document, entry, "en");
document["image_url"].Should().Equals("http://myimage");
}
private void AddSearchDocumentInternal(SearchDocument searchDocument, bool autoCommit)
{
Requires.NotNull("SearchDocument", searchDocument);
Requires.NotNullOrEmpty("UniqueKey", searchDocument.UniqueKey);
Requires.NotNegative("SearchTypeId", searchDocument.SearchTypeId);
Requires.PropertyNotEqualTo("searchDocument", "SearchTypeId", searchDocument.SearchTypeId, 0);
Requires.PropertyNotEqualTo("searchDocument", "ModifiedTimeUtc", searchDocument.ModifiedTimeUtc.ToString(CultureInfo.InvariantCulture), DateTime.MinValue.ToString(CultureInfo.InvariantCulture));
if (searchDocument.SearchTypeId == this._moduleSearchTypeId)
{
if (searchDocument.ModuleDefId <= 0)
{
throw new ArgumentException(Localization.GetExceptionMessage("ModuleDefIdMustBeGreaterThanZero", "ModuleDefId must be greater than zero when SearchTypeId is for a module"));
}
if (searchDocument.ModuleId <= 0)
{
throw new ArgumentException(Localization.GetExceptionMessage("ModuleIdMustBeGreaterThanZero", "ModuleId must be greater than zero when SearchTypeId is for a module"));
}
}
else
{
if (searchDocument.ModuleDefId > 0)
{
throw new ArgumentException(Localization.GetExceptionMessage("ModuleDefIdWhenSearchTypeForModule", "ModuleDefId is needed only when SearchTypeId is for a module"));
}
if (searchDocument.ModuleId > 0)
{
throw new ArgumentException(Localization.GetExceptionMessage("ModuleIdWhenSearchTypeForModule", "ModuleId is needed only when SearchTypeId is for a module"));
}
}
var doc = new Document();
var sb = new StringBuilder();
// TODO - Set setOmitTermFreqAndPositions
// TODO - Check if Numeric fields need to have Store.YES or Store.NO
// TODO - Should ModifiedTime be mandatory
// TODO - Is Locale needed for a single-language site
// TODO - Sorting
// Field.Store.YES | Stores the value. When the value is stored, the original String in its entirety is recorded in the index
// Field.Store.NO | Doesn’t store the value. This option is often used along with Index.ANALYZED to index a large text field that doesn’t need to be retrieved
// | in its original form, such as bodies of web pages, or any other type of text document.
// Index.ANALYZED | Use the analyzer to break the field’s value into a stream of separate tokens and make each token searchable
// Index.NOT_ANALYZED | Do index the field, but don’t analyze the String value.Instead, treat the Field’s entire value as a single token and make that token searchable.
// Generic and Additional SearchDocument Params
this.AddSearchDocumentParamters(doc, searchDocument, sb);
// Remove the existing document from Lucene
this.DeleteSearchDocumentInternal(searchDocument, false);
// Add only when Document is active. The previous call would have otherwise deleted the document if it existed earlier
if (searchDocument.IsActive)
{
Thread.SetData(Thread.GetNamedDataSlot(Constants.TlsSearchInfo), searchDocument);
try
{
LuceneController.Instance.Add(doc);
}
finally
{
Thread.SetData(Thread.GetNamedDataSlot(Constants.TlsSearchInfo), null);
}
}
if (autoCommit)
{
this.Commit();
}
}
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddCreated()
{
var entry = GetCatalogEntryRow("Product");
var document = new SearchDocument();
_subject.UpdateSearchDocument(ref document, entry, "en");
document["created"].Should().Equals(new DateTime(2012, 4, 4).ToString("yyyyMMddhhmmss"));
}
private void AddSearchDocumentParamters(Document doc, SearchDocument searchDocument, StringBuilder sb)
{
// mandatory fields
doc.Add(new Field(Constants.UniqueKeyTag, SearchHelper.Instance.StripTagsNoAttributes(searchDocument.UniqueKey, true), Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new NumericField(Constants.PortalIdTag, Field.Store.YES, true).SetIntValue(searchDocument.PortalId));
doc.Add(new NumericField(Constants.SearchTypeTag, Field.Store.YES, true).SetIntValue(searchDocument.SearchTypeId));
doc.Add(!string.IsNullOrEmpty(searchDocument.CultureCode)
? new NumericField(Constants.LocaleTag, Field.Store.YES, true).SetIntValue(Localization.GetCultureLanguageID(searchDocument.CultureCode))
: new NumericField(Constants.LocaleTag, Field.Store.YES, true).SetIntValue(-1));
if (!string.IsNullOrEmpty(searchDocument.Title))
{
var field = new Field(Constants.TitleTag, StripTagsRetainAttributes(searchDocument.Title, HtmlAttributesToRetain, false, true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
if (this._titleBoost > 0 && this._titleBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._titleBoost / 10f;
}
doc.Add(field);
}
if (!string.IsNullOrEmpty(searchDocument.Description))
{
var field = new Field(Constants.DescriptionTag, StripTagsRetainAttributes(searchDocument.Description, HtmlAttributesToRetain, false, true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
if (this._descriptionBoost > 0 && this._descriptionBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._descriptionBoost / 10f;
}
doc.Add(field);
}
if (!string.IsNullOrEmpty(searchDocument.Body))
{
doc.Add(new Field(Constants.BodyTag, StripTagsRetainAttributes(searchDocument.Body, HtmlAttributesToRetain, false, true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
}
if (!string.IsNullOrEmpty(searchDocument.Url))
{
doc.Add(new Field(Constants.UrlTag, SearchHelper.Instance.StripTagsNoAttributes(searchDocument.Url, true), Field.Store.YES, Field.Index.NOT_ANALYZED));
}
if (!string.IsNullOrEmpty(searchDocument.QueryString))
{
doc.Add(new Field(Constants.QueryStringTag, searchDocument.QueryString, Field.Store.YES, Field.Index.NOT_ANALYZED));
}
foreach (var kvp in searchDocument.Keywords)
{
var key = kvp.Key.ToLowerInvariant();
var needAnalyzed = Constants.FieldsNeedAnalysis.Contains(key);
var field = new Field(SearchHelper.Instance.StripTagsNoAttributes(Constants.KeywordsPrefixTag + kvp.Key, true), SearchHelper.Instance.StripTagsNoAttributes(kvp.Value, true), Field.Store.YES, needAnalyzed ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED);
switch (key)
{
case Constants.TitleTag:
if (this._titleBoost > 0 && this._titleBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._titleBoost / 10f;
}
break;
case Constants.SubjectTag:
if (this._contentBoost > 0 && this._contentBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._contentBoost / 10f;
}
break;
case Constants.CommentsTag:
if (this._descriptionBoost > 0 && this._descriptionBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._descriptionBoost / 10f;
}
break;
case Constants.AuthorNameTag:
if (this._authorBoost > 0 && this._authorBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._authorBoost / 10f;
}
break;
}
doc.Add(field);
sb.Append(SearchHelper.Instance.StripTagsNoAttributes(kvp.Value, true)).Append(" ");
}
foreach (var kvp in searchDocument.NumericKeys)
{
doc.Add(new NumericField(SearchHelper.Instance.StripTagsNoAttributes(Constants.NumericKeyPrefixTag + kvp.Key, true), Field.Store.YES, true).SetIntValue(kvp.Value));
}
bool tagBoostApplied = false;
foreach (var tag in searchDocument.Tags)
{
var field = new Field(Constants.Tag, SearchHelper.Instance.StripTagsNoAttributes(tag.ToLowerInvariant(), true), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
if (!tagBoostApplied)
{
if (this._tagBoost > 0 && this._tagBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._tagBoost / 10f;
tagBoostApplied = true;
}
}
doc.Add(field);
}
AddIntField(doc, searchDocument.TabId, Constants.TabIdTag);
AddIntField(doc, searchDocument.ModuleDefId, Constants.ModuleDefIdTag);
AddIntField(doc, searchDocument.ModuleId, Constants.ModuleIdTag);
AddIntField(doc, searchDocument.AuthorUserId, Constants.AuthorIdTag);
AddIntField(doc, searchDocument.RoleId, Constants.RoleIdTag);
if (searchDocument.AuthorUserId > 0)
{
var user = UserController.Instance.GetUserById(searchDocument.PortalId, searchDocument.AuthorUserId);
if (user != null && !string.IsNullOrEmpty(user.DisplayName))
{
var field = new Field(Constants.AuthorNameTag, user.DisplayName, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
if (this._authorBoost > 0 && this._authorBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._authorBoost / 10f;
}
doc.Add(field);
}
}
if (!string.IsNullOrEmpty(searchDocument.Permissions))
{
doc.Add(new Field(Constants.PermissionsTag, SearchHelper.Instance.StripTagsNoAttributes(searchDocument.Permissions, true), Field.Store.YES, Field.Index.NOT_ANALYZED));
}
doc.Add(new NumericField(Constants.ModifiedTimeTag, Field.Store.YES, true).SetLongValue(long.Parse(searchDocument.ModifiedTimeUtc.ToString(Constants.DateTimeFormat))));
if (sb.Length > 0)
{
var field = new Field(Constants.ContentTag, SearchHelper.Instance.StripTagsNoAttributes(sb.ToString(), true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
doc.Add(field);
if (this._contentBoost > 0 && this._contentBoost != Constants.StandardLuceneBoost)
{
field.Boost = this._contentBoost / 10f;
}
}
}
public IList <SearchDocument> Search <T>(IList <Filter> searchFilters, IList <SearchSortField> sortFields) where T : class
{
var connection = new ConnectionSettings(new Uri("http://localhost:9200"));
connection.DefaultIndex("search_example");
var filter = CreateFilter(searchFilters);
var searchRequest = new SearchRequest <DeveloperModel>
{
Query = new BoolQuery {
Filter = filter
},
Size = 100,
From = 0
};
var client = new ElasticClient(connection);
using (var stream = new MemoryStream())
{
client.Serializer.Serialize(searchRequest, stream);
string searchJson = Encoding.UTF8.GetString(stream.ToArray());
System.Diagnostics.Trace.WriteLine(searchJson);
}
var response = client.Search <DeveloperModel>(searchRequest);
var result = new List <SearchDocument>();
if (response.IsValid)
{
foreach (var model in response.Documents)
{
var doc = new SearchDocument
{
Id = model.Id,
Fields = new List <SearchDocumentField>
{
new SearchDocumentField {
FieldName = "Id", Value = model.Id
},
new SearchDocumentField {
FieldName = "City", Value = model.City
},
new SearchDocumentField {
FieldName = "HouseNo", Value = model.HouseNo
},
new SearchDocumentField {
FieldName = "Name", Value = model.Name
},
new SearchDocumentField {
FieldName = "Street", Value = model.Street
}
}
};
result.Add(doc);
}
}
return(result);
}
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddContentLink()
{
var entry = GetCatalogEntryRow("Product");
var document = new SearchDocument();
_subject.UpdateSearchDocument(ref document, entry, "en");
document["content_link"].Should().Equals(GetContentReference(444, CatalogContentType.CatalogEntry).ToString());
}
private async Task UpdateIndicesWithAddedDocuments(DateTimeOffset lastSyncPoint, DateTimeOffset currentSyncPoint, int numberOfUpdates, IndexWriter indexWriter, CancellationToken cancellationToken)
{
using (var dbConnection = await _SQLservice.GetConnection(cancellationToken))
{
var startRow = 1;
var dbCountCommand = @"SELECT Id, ISNULL(Name,''), ISNULL(Content,''), UpdatedAt from [dbo].[Test_Data]
WHERE UpdatedAt >= @lastSyncPoint AND UpdatedAt < @currentSyncPoint AND DeletedAt IS NULL
ORDER BY Id ASC OFFSET @StartRow - 1 ROWS FETCH NEXT @RowsPerPage ROWS ONLY ";
while (numberOfUpdates >= startRow)
{
cancellationToken.ThrowIfCancellationRequested();
var cmd = new SqlCommand(dbCountCommand, dbConnection);
cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint;
cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset);
cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint;
cmd.Parameters.Add("@StartRow", System.Data.SqlDbType.Int);
cmd.Parameters["@StartRow"].Value = startRow;
cmd.Parameters.Add("@RowsPerPage", System.Data.SqlDbType.Int);
cmd.Parameters["@RowsPerPage"].Value = PageSize;
try
{
using (var reader = await _SQLservice.ExecuteReaderWithRetryAsync(cmd, System.Data.CommandBehavior.SequentialAccess, cancellationToken))
{
while (await reader.ReadAsync())
{
var document_id = await reader.GetFieldValueAsync <int>(0);
var document_name = await reader.GetFieldValueAsync <string>(1);
var document_content = await reader.GetFieldValueAsync <string>(2);
var document_updatedAt = await reader.GetFieldValueAsync <DateTimeOffset>(3);
var updatedAtAsNumber = int.Parse(document_updatedAt.ToString("yyyyMMdd"));
var searchDocument = new SearchDocument()
{
DocumentID = document_id.ToString(),
Name = document_name,
Content = document_content,
UpdatedAt = updatedAtAsNumber
};
var doc = new Lucene.Net.Documents.Document
{
// StringField indexes but doesn't tokenize
new Lucene.Net.Documents.StringField("doc_id", searchDocument.DocumentID, Lucene.Net.Documents.Field.Store.YES),
new Lucene.Net.Documents.StringField("name", searchDocument.Name, Lucene.Net.Documents.Field.Store.YES),
new Lucene.Net.Documents.TextField("content", searchDocument.Content, Lucene.Net.Documents.Field.Store.YES),
new Lucene.Net.Documents.Int32Field("updated", searchDocument.UpdatedAt, Lucene.Net.Documents.Field.Store.YES)
};
indexWriter.AddDocument(doc);
startRow++;
}
}
_logger.LogDebug("Processed {0} records (of {1} total) for update", (startRow - 1), numberOfUpdates);
}
catch (Exception ex)
{
_logger.LogError(ex, "unexpected failure during indexes update");
throw;
}
}
_logger.LogInformation("Processed {0} records for update of FTS indices. Completed.", (startRow - 1));
}
}
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddDisplayName()
{
var entry = GetCatalogEntryRow("Product");
var document = new SearchDocument();
_subject.UpdateSearchDocument(ref document, entry, "en");
document["displayname"].Should().Equals("DisplayName");
}
public JObject IngestESData(SearchDocument document, string token)
{
return(_bomSearchDataAccess.IngestESData(document, token));
}
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddTopCategory()
{
var entry = GetCatalogEntryRow("Product");
var document = new SearchDocument();
_subject.UpdateSearchDocument(ref document, entry, "en");
document["top_category_name"].Should().Equals("Catalog");
}
public IHttpActionResult SearchDocuments(SearchDocument objS)
{
TranslatorBusinessService objTBS = new TranslatorBusinessService();
return(Ok(objTBS.Documents(objS)));
}
