public IList<SearchDocument> Search(IList<Filter> searchFilters, IList<SearchSortField> sortFields) { if (searchFilters == null || searchFilters.Count == 0) return new List<SearchDocument>(); var sort = CreateSort(searchFilters, sortFields); var reader = IndexReader.Open(_directory, true); var searcher = new IndexSearcher(reader); var query = CreateQuery(searchFilters); var hits = searcher.Search(query, new QueryWrapperFilter(query), 100, sort); var searchResult = new List<SearchDocument>(); foreach (var facet in hits.ScoreDocs) { var doc = searcher.Doc(facet.Doc); var searchDoc = new SearchDocument { Fields = new List<SearchDocumentField>() }; var docFields = doc.GetFields(); foreach (var field in docFields) { var value = doc.Get(field.Name); searchDoc.Fields.Add(new SearchDocumentField {FieldName = field.Name, Value = value}); } searchResult.Add(searchDoc); } return searchResult; }
public void Save(SearchDocument doc) { if (doc == null) throw new ArgumentNullException(nameof(doc)); if (doc.Fields == null || doc.Fields.Count == 0) return; var document = new Document(); foreach (var docField in doc.Fields) { if (string.IsNullOrWhiteSpace(docField.FieldName)) throw new Exception("Field name cannot be empty"); if (string.IsNullOrWhiteSpace(docField.Value)) continue; var field = new Field(docField.FieldName, docField.Value, Field.Store.YES, Field.Index.ANALYZED); document.Add(field); } var writer = new IndexWriter( _directory, new StandardAnalyzer(global::Lucene.Net.Util.Version.LUCENE_30), !_created, IndexWriter.MaxFieldLength.UNLIMITED); using (writer) { writer.AddDocument(document); } _created = true; }
/// <summary> /// Called when a catalog entry is indexed. /// We use this method to load the prices for the variants of a product and store /// the highest variant price on the product for optimal retrieval in the product listing. /// </summary> protected override void OnCatalogEntryIndex(ref SearchDocument document, CatalogEntryDto.CatalogEntryRow entry, string language) { if (!entry.ClassTypeId.Equals(EntryType.Product)) { return; } UpdateSearchDocument(ref document, entry, language); }
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddColor() { var entry = GetCatalogEntryRow("Product"); var document = new SearchDocument(); _subject.UpdateSearchDocument(ref document, entry, "en"); document["color"].Should().Equals("Green"); }
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddOriginalUSDPrice() { var entry = GetCatalogEntryRow("Product"); var document = new SearchDocument(); _subject.UpdateSearchDocument(ref document, entry, "en"); document[IndexingHelper.GetOriginalPriceField(MarketId.Default, Currency.USD)].Value.ToString() .Should() .Equals((1000m).ToString(CultureInfo.InvariantCulture)); }
public void Save(string filename, SearchDocument doc) { if (doc == null) return; var ws = new XmlWriterSettings(); ws.Indent = true; ws.IndentChars = " "; ws.CheckCharacters = true; var serializer = new XmlSerializer(typeof(SearchDocumentDTO), GetUniverse()); using (var writer = XmlWriter.Create(filename, ws)) serializer.Serialize(writer, _assembler.Create(doc)); }
public void UpdateSearchDocument(ref SearchDocument document, CatalogEntryDto.CatalogEntryRow entry, string language) { var sw = new Stopwatch(); sw.Start(); var contentLink = _referenceConverter.GetContentLink(entry.Code); var productContent = _contentLoader.Get<FashionProduct>(contentLink); var variants = _contentLoader.GetItems(productContent.GetVariants(_relationRepository), CultureInfo.GetCultureInfo(language)).OfType<FashionVariant>().ToList(); AddPrices(document, variants); AddColors(document, variants); AddSizes(document, variants); AddCodes(document, variants); document.Add(new SearchField("code", productContent.Code, new[] { SearchField.Store.YES, SearchField.IncludeInDefaultSearch.YES })); document.Add(new SearchField("displayname", productContent.DisplayName)); document.Add(new SearchField("image_url", _assetUrlResolver.GetAssetUrl<IContentImage>(productContent))); document.Add(new SearchField("content_link", productContent.ContentLink.ToString())); document.Add(new SearchField("created", productContent.Created.ToString("yyyyMMddhhmmss"))); document.Add(new SearchField("brand", productContent.Brand)); document.Add(new SearchField("top_category_name", GetTopCategory(productContent).DisplayName)); sw.Stop(); _log.Debug(string.Format("Indexing of {0} for {1} took {2}", productContent.Code, language, sw.Elapsed.Milliseconds)); }
protected override void OnCatalogEntryIndex(ref SearchDocument document, CatalogEntryDto.CatalogEntryRow entry, string language) { if (entry.ClassTypeId != "Product") { return; } var result = GetDocument(language, entry.Code).Result; if (result == null) { throw new Exception(String.Format("could not connect to {0}, please make sure site is active", _url + String.Format("referenceapi/searchdocuments/{0}/{1}", language, entry.Code))); } foreach (var field in result.Fields.Where(field => field.Values.Any())) { document.Add ( field.IsDecimal ? new SearchField(field.Name, Decimal.Parse(field.Values.First(), CultureInfo.InvariantCulture), field.Attributes.ToArray()) : new SearchField(field.Name, field.Values.First(), field.Attributes.ToArray()) ); } }
private void DeleteSearchDocumentInternal(SearchDocument searchDocument, bool autoCommit) { var query = new BooleanQuery(); if (searchDocument.SearchTypeId > -1) { query.Add(NumericValueQuery(Constants.SearchTypeTag, searchDocument.SearchTypeId), Occur.MUST); } if (searchDocument.PortalId > -1) { query.Add(NumericValueQuery(Constants.PortalIdTag, searchDocument.PortalId), Occur.MUST); } if (searchDocument.RoleId > -1) { query.Add(NumericValueQuery(Constants.RoleIdTag, searchDocument.RoleId), Occur.MUST); } if (searchDocument.ModuleDefId > 0) { query.Add(NumericValueQuery(Constants.ModuleDefIdTag, searchDocument.ModuleDefId), Occur.MUST); } if (searchDocument.ModuleId > 0) { query.Add(NumericValueQuery(Constants.ModuleIdTag, searchDocument.ModuleId), Occur.MUST); } if (searchDocument.TabId > 0) { query.Add(NumericValueQuery(Constants.TabIdTag, searchDocument.TabId), Occur.MUST); } if (searchDocument.AuthorUserId > 0) { query.Add(NumericValueQuery(Constants.AuthorIdTag, searchDocument.AuthorUserId), Occur.MUST); } if (!string.IsNullOrEmpty(searchDocument.UniqueKey)) { query.Add(new TermQuery(new Term(Constants.UniqueKeyTag, searchDocument.UniqueKey)), Occur.MUST); } if (!string.IsNullOrEmpty(searchDocument.QueryString)) { query.Add(new TermQuery(new Term(Constants.QueryStringTag, searchDocument.QueryString)), Occur.MUST); } if (!string.IsNullOrEmpty(searchDocument.CultureCode)) { query.Add(NumericValueQuery(Constants.LocaleTag, Localization.GetCultureLanguageID(searchDocument.CultureCode)), Occur.MUST); } LuceneController.Instance.Delete(query); if (autoCommit) { this.Commit(); } }
public void DeleteSearchDocument(SearchDocument searchDocument) { this.DeleteSearchDocumentInternal(searchDocument, false); }
public void GetSearchResultsDetailed() { const string keyword = "super"; const string moduleBody = "super content is here"; const string userUrl = "mysite/userid/1"; const string tabUrl1 = "mysite/Home"; const string tabUrl2 = "mysite/AboutUs"; //first tab with 2 modules var doc1 = new SearchDocument { UniqueKey = "key01", TabId = TabId1, Url = tabUrl1, Title = keyword, SearchTypeId = TabSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow }; var doc2 = new SearchDocument { UniqueKey = "key02", TabId = TabId1, Title = keyword, Url = tabUrl1, SearchTypeId = ModuleSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, ModuleDefId = HtmlModuleDefId, ModuleId = HtmlModuleId2, Body = moduleBody, RoleId = 731 }; var doc3 = new SearchDocument { UniqueKey = "key03", TabId = TabId1, Title = keyword, Url = tabUrl1, SearchTypeId = ModuleSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, ModuleDefId = HtmlModuleDefId, ModuleId = HtmlModuleId1, Body = moduleBody, RoleId = 731 }; //second tab with 1 module var doc4 = new SearchDocument { UniqueKey = "key04", TabId = TabId2, Url = tabUrl2, Title = keyword, SearchTypeId = TabSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, RoleId = RoleId0 }; var doc5 = new SearchDocument { UniqueKey = "key05", TabId = TabId2, Title = keyword, Url = tabUrl2, SearchTypeId = ModuleSearchTypeId, ModuleDefId = HtmlModuleId, ModuleId = HtmlModuleId3, ModifiedTimeUtc = DateTime.UtcNow, Body = moduleBody, RoleId = 731 }; //user doc var userdoc = new SearchDocument { UniqueKey = "key06", Url = userUrl, Title = keyword, SearchTypeId = UserSearchTypeId, ModifiedTimeUtc = DateTime.UtcNow, RoleId = RoleId731 }; _internalSearchController.AddSearchDocument(doc1); _internalSearchController.AddSearchDocument(doc2); _internalSearchController.AddSearchDocument(doc3); _internalSearchController.AddSearchDocument(doc4); _internalSearchController.AddSearchDocument(doc5); _internalSearchController.AddSearchDocument(userdoc); var query = new SearchQuery { KeyWords = keyword, SearchTypeIds = new[] { ModuleSearchTypeId, TabSearchTypeId, UserSearchTypeId }, RoleId = 731 }; //Run var search = GetGroupedDetailViewResults(query); //Assert var groupedDetailViews = search as List <GroupedDetailView> ?? search.ToList(); //Overall 3 groups - tab1, tab2 and user Assert.AreEqual(3, groupedDetailViews.Count()); //Tab 1 has 2 DetailViews Assert.AreEqual(2, groupedDetailViews.Single(x => x.DocumentUrl == tabUrl1).Results.Count()); //Tab 2 has 1 DetailViews Assert.AreEqual(1, groupedDetailViews.Single(x => x.DocumentUrl == tabUrl2).Results.Count()); //UserUrl has 1 DetailViews Assert.AreEqual(1, groupedDetailViews.Single(x => x.DocumentUrl == userUrl).Results.Count()); }
private String DoProductIdx(DotNetNuke.Entities.Portals.PortalInfo portal, DateTime lastrun, Boolean debug) { if (debug) { InternalSearchController.Instance.DeleteAllDocuments(portal.PortalID, SearchHelper.Instance.GetSearchTypeByName("tab").SearchTypeId); } var searchDocs = new List <SearchDocument>(); var culturecodeList = DnnUtils.GetCultureCodeList(portal.PortalID); var storeSettings = new StoreSettings(portal.PortalID); foreach (var lang in culturecodeList) { var strContent = ""; // select all products var objCtrl = new NBrightBuyController(); var strFilter = " and NB1.ModifiedDate > convert(datetime,'" + lastrun.ToString("s") + "') "; if (debug) { strFilter = ""; } var l = objCtrl.GetList(portal.PortalID, -1, "PRD", strFilter); foreach (var p in l) { var prodData = new ProductData(p.ItemID, lang); strContent = prodData.Info.GetXmlProperty("genxml/textbox/txtproductref") + " : " + prodData.SEODescription + " " + prodData.SEOName + " " + prodData.SEOTagwords + " " + prodData.SEOTitle; if (strContent != "") { var tags = new List <String>(); tags.Add("nbsproduct"); //Get the description string string strDescription = HtmlUtils.Shorten(HtmlUtils.Clean(strContent, false), 100, "..."); var searchDoc = new SearchDocument(); // Assigns as a Search key the SearchItems' searchDoc.UniqueKey = prodData.Info.ItemID.ToString(""); searchDoc.QueryString = "ref=" + prodData.Info.GetXmlProperty("genxml/textbox/txtproductref"); searchDoc.Title = prodData.ProductName; searchDoc.Body = strContent; searchDoc.Description = strDescription; if (debug) { searchDoc.ModifiedTimeUtc = DateTime.Now.Date; } else { searchDoc.ModifiedTimeUtc = prodData.Info.ModifiedDate; } searchDoc.AuthorUserId = 1; searchDoc.TabId = storeSettings.ProductDetailTabId; searchDoc.PortalId = portal.PortalID; searchDoc.SearchTypeId = SearchHelper.Instance.GetSearchTypeByName("tab").SearchTypeId; searchDoc.CultureCode = lang; searchDoc.Tags = tags; //Add Module MetaData searchDoc.ModuleDefId = 0; searchDoc.ModuleId = 0; searchDocs.Add(searchDoc); } } } //Index InternalSearchController.Instance.AddSearchDocuments(searchDocs); InternalSearchController.Instance.Commit(); return(" - NBS-DNNIDX scheduler ACTIVATED "); }
public virtual SearchDocument GetDocFromContent(IContent content) { try { var document = new Document(); IncludeDefaultField(document, content); var possibleIndexModels = LuceneConfiguration.IncludedTypes.Where(x => x.Value.ContentType.IsAssignableFrom(content.GetOriginalType())).Select(x => x.Value).ToList(); foreach (var documentIndexModel in possibleIndexModels) { IncludeContentField(document, content, documentIndexModel); var computedFieldList = documentIndexModel.IndexedFields; foreach (var field in computedFieldList) { var fieldInstance = field.Value; var value = fieldInstance.GetValue(content, field.Key); if (value != null) { AbstractField luceneField; var indexFieldName = ContentIndexHelpers.GetIndexFieldName(field.Key); var existedFieldName = document.GetField(indexFieldName); if (existedFieldName != null) { document.RemoveField(indexFieldName); } if (fieldInstance.DataType == LuceneFieldType.Multilist) { var listValue = (List <string>)value; foreach (var item in listValue) { document.Add(new Field(indexFieldName, item, fieldInstance.Store, fieldInstance.Index, fieldInstance.Vector)); } continue; } switch (fieldInstance.DataType) { case LuceneFieldType.Datetime: DateTime d1 = Convert.ToDateTime(value); luceneField = new Field(indexFieldName, DateTools.DateToString(d1.ToUniversalTime(), DateTools.Resolution.SECOND), fieldInstance.Store, fieldInstance.Index, fieldInstance.Vector); break; case LuceneFieldType.Numeric: luceneField = new NumericField(indexFieldName, fieldInstance.Store, true).SetLongValue(string.IsNullOrEmpty(value + "") ? 0 : long.Parse(value + "")); break; default: luceneField = new Field(indexFieldName, value.ToString(), fieldInstance.Store, fieldInstance.Index, fieldInstance.Vector); break; } document.Add(luceneField); } } } var result = new SearchDocument() { Id = SearchDocument.FormatDocumentId(content), Document = document }; return(result); } catch (Exception ex) { _logger.Error("Fetch Document error", ex); return(null); } }
/// <summary> /// Parse JSON into a SearchDocument. /// </summary> /// <param name="reader">The JSON reader.</param> /// <param name="typeToConvert">The type to convert to.</param> /// <param name="options">Serialization options.</param> /// <param name="recursionDepth"> /// Depth of the current read recursion to bail out of circular /// references. /// </param> /// <returns>A deserialized SearchDocument.</returns> public static SearchDocument ReadSearchDocument( ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, int?recursionDepth = null) { Debug.Assert(typeToConvert != null); Debug.Assert(typeToConvert.IsAssignableFrom(typeof(SearchDocument))); recursionDepth ??= options.GetMaxRecursionDepth(); AssertRecursionDepth(recursionDepth.Value); SearchDocument doc = new SearchDocument(); reader.Expects(JsonTokenType.StartObject); while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) { string propertyName = reader.ExpectsPropertyName(); // Ignore OData properties - we don't expose those on custom // user schemas if (!propertyName.StartsWith(Constants.ODataKeyPrefix, StringComparison.OrdinalIgnoreCase)) { object value = ReadSearchDocObject(ref reader, recursionDepth.Value - 1); doc[propertyName] = value; } else { reader.Skip(); } } return(doc); // TODO: #10596 - Investigate using JsonSerializer for reading SearchDocument properties // The built in converters for JsonSerializer are a little more // helpful than we want right now and will do things like turn "1" // to the integer 1 instead of a string. The number of special // cases needed for converting dynamic documents is small enough // that we're hard coding them here for now. We'll revisit with // Search experts and their customer scenarios to get this right in // the next preview. object ReadSearchDocObject(ref Utf8JsonReader reader, int depth) { AssertRecursionDepth(depth); switch (reader.TokenType) { case JsonTokenType.String: case JsonTokenType.Number: case JsonTokenType.True: case JsonTokenType.False: case JsonTokenType.None: case JsonTokenType.Null: return(ReadPrimitiveValue(ref reader)); case JsonTokenType.StartObject: // TODO: #10592- Unify on an Azure.Core spatial type // Return a complex object return(ReadSearchDocument(ref reader, typeof(SearchDocument), options, depth - 1)); case JsonTokenType.StartArray: var list = new List <object>(); while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { recursionDepth--; object value = ReadSearchDocObject(ref reader, depth - 1); list.Add(value); } return(list.ToArray()); default: throw new JsonException(); } } }
public void SearchDocumentsTest() { var locationData = new LocationData(ConfigurationManager.AppSettings[ConfigurationKeys.BingMapsKey]); var searchServiceName = ConfigurationManager.AppSettings[ConfigurationKeys.SearchServiceName]; var searchServiceKey = ConfigurationManager.AppSettings[ConfigurationKeys.SearchServiceKey]; var searchClient = new SearchDocument(searchServiceName, searchServiceKey); var data = locationData.GetBoundingBoxCoordinates("Delhi,India").Result; var searchResult = searchClient.SearchDocuments <GpsSensorRecord>("*", SearchDocument.FilterTextForLocationBounds("geoCoordinates", data)); var locationPoint = new LocationPoint { Latitude = 28.644979476928711, Longitude = 77.2164306640625 }; var searchByDistance = searchClient.SearchDocuments <GpsSensorRecord>("*", SearchDocument.FilterTextForDistanceFromPoint("geoCoordinates", locationPoint, 100)); Assert.IsTrue(searchResult.Any()); Assert.IsTrue(searchByDistance.Any()); var dataUsa = locationData.GetBoundingBoxCoordinates("Washington,USA").Result; var searchResultUsa = searchClient.SearchDocuments <GpsSensorRecord>("*", SearchDocument.FilterTextForLocationBounds("geoCoordinates", dataUsa)); Assert.IsTrue(searchResultUsa.Any()); }
/// <summary> /// Geocodes and returns the result. /// </summary> /// <param name="street"></param> /// <param name="houseNumber"></param> /// <param name="country"></param> /// <param name="postalCode"></param> /// <param name="commune"></param> /// <returns></returns> public IGeoCoderResult Code( string country, string postalCode, string commune, string street, string houseNumber) { // build the request url. var builder = new StringBuilder(); builder.Append(street); builder.Append(" "); builder.Append(houseNumber); builder.Append(" "); builder.Append(postalCode); builder.Append(" "); builder.Append(commune); builder.Append(" "); builder.Append(country); builder.Append(" "); string url = string.Format(System.Globalization.CultureInfo.InvariantCulture, _geocodingUrl + "&format=xml&polygon=1&addressdetails=1", builder); // create the source and get the xml. IXmlSource source = this.DownloadXml(url); // create the kml. var search_doc = new SearchDocument(source); // check if there are responses. var res = new GeoCoderResult(); res.Accuracy = AccuracyEnum.UnkownLocationLevel; if (search_doc.Search is searchresults) { searchresults result_v1 = search_doc.Search as searchresults; if (result_v1.place != null && result_v1.place.Length > 0) { double latitude; double longitude; if (double.TryParse(result_v1.place[0].lat, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out latitude) && double.TryParse(result_v1.place[0].lon, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out longitude)) { res.Latitude = latitude; res.Longitude = longitude; res.Text = result_v1.place[0].display_name; switch (result_v1.place[0].@class) { case "place": switch (result_v1.place[0].type) { case "town": res.Accuracy = AccuracyEnum.TownLevel; break; case "house": res.Accuracy = AccuracyEnum.AddressLevel; break; } break; case "highway": res.Accuracy = AccuracyEnum.StreetLevel; break; case "boundary": res.Accuracy = AccuracyEnum.PostalCodeLevel; break; } } } } else if (search_doc.Search is OsmSharp.Xml.Nominatim.Reverse.v1.reversegeocode) { reversegeocode result_v1 = search_doc.Search as OsmSharp.Xml.Nominatim.Reverse.v1.reversegeocode; if (result_v1.result != null && result_v1.result.Length > 0) { double latitude; double longitude; if (double.TryParse(result_v1.result[0].lat, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out latitude) && double.TryParse(result_v1.result[0].lon, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out longitude)) { res.Latitude = latitude; res.Longitude = longitude; res.Text = result_v1.result[0].Value; res.Accuracy = AccuracyEnum.UnkownLocationLevel; } } } return(res); }
/// ----------------------------------------------------------------------------- /// <summary> /// Returns the collection of SearchDocuments populated with Tab MetaData for the given portal. /// </summary> /// <param name="portalId"></param> /// <param name="startDate"></param> /// <returns></returns> /// <history> /// [vnguyen] 04/16/2013 created /// </history> /// ----------------------------------------------------------------------------- public override IEnumerable <SearchDocument> GetSearchDocuments(int portalId, DateTime startDate) { var searchDocuments = new Dictionary <string, SearchDocument>(); var needReindex = PortalController.GetPortalSettingAsBoolean(UserIndexResetFlag, portalId, false); if (needReindex) { startDate = SqlDateTime.MinValue.Value; } try { var startUserId = Null.NullInteger; while (true) { var reader = DataProvider.Instance().GetAvailableUsersForIndex(portalId, startDate, startUserId, BatchSize); int rowsAffected = 0; var indexedUsers = new List <int>(); while (reader.Read()) { var userId = Convert.ToInt32(reader["UserId"]); var displayName = reader["DisplayName"].ToString(); var firstName = reader["FirstName"].ToString(); var propertyName = reader["PropertyName"].ToString(); var propertyValue = reader["PropertyValue"].ToString(); var visibilityMode = ((UserVisibilityMode)Convert.ToInt32(reader["Visibility"])); var modifiedTime = Convert.ToDateTime(reader["ModifiedTime"]).ToUniversalTime(); //log the userid so that it can get the correct user collection next time. if (userId > startUserId) { startUserId = userId; } var uniqueKey = string.Format("{0}_{1}", userId, visibilityMode).ToLowerInvariant(); if (visibilityMode == UserVisibilityMode.FriendsAndGroups) { uniqueKey = string.Format("{0}_{1}", uniqueKey, reader["ExtendedVisibility"]); } if (searchDocuments.ContainsKey(uniqueKey)) { var document = searchDocuments[uniqueKey]; document.Body += string.Format(" {0}", propertyValue); if (modifiedTime > document.ModifiedTimeUtc) { document.ModifiedTimeUtc = modifiedTime; } } else { //Need remove use exists index for all visibilities. if (!indexedUsers.Contains(userId)) { indexedUsers.Add(userId); } if (!string.IsNullOrEmpty(propertyValue)) { var searchDoc = new SearchDocument { SearchTypeId = UserSearchTypeId, UniqueKey = uniqueKey, PortalId = portalId, ModifiedTimeUtc = modifiedTime, Body = propertyValue, Description = firstName, Title = displayName }; searchDocuments.Add(uniqueKey, searchDoc); } else if (!searchDocuments.ContainsKey(string.Format("{0}_{1}", userId, UserVisibilityMode.AllUsers).ToLowerInvariant())) { if (!indexedUsers.Contains(userId)) { indexedUsers.Add(userId); } //if the user doesn't exist in search collection, we need add it with ALLUsers mode, //so that can make sure DisplayName will be indexed var searchDoc = new SearchDocument { SearchTypeId = UserSearchTypeId, UniqueKey = string.Format("{0}_{1}", userId, UserVisibilityMode.AllUsers).ToLowerInvariant(), PortalId = portalId, ModifiedTimeUtc = modifiedTime, Body = string.Empty, Description = firstName, Title = displayName }; searchDocuments.Add(searchDoc.UniqueKey, searchDoc); } } rowsAffected++; } //close the data reader reader.Close(); //remov exists indexes DeleteDocuments(portalId, indexedUsers); if (rowsAffected == 0) { break; } } if (needReindex) { PortalController.DeletePortalSetting(portalId, UserIndexResetFlag); } } catch (Exception ex) { Exceptions.Exceptions.LogException(ex); } return(searchDocuments.Values); }
private static SearchDocument CreateSearchDocument(ModuleInfo modInfo, OpenContentSettings settings, JToken contentData, string itemId, string culture, string dataItemTitle, DateTime time) { // existance of settings.Template.Main has already been checked: we wouldn't be here if it doesn't exist // but still, we don't want to count on that too much var ps = new PortalSettings(modInfo.PortalID); ps.PortalAlias = PortalAliasController.Instance.GetPortalAlias(ps.DefaultPortalAlias); string url = null; // Check if it is a single or list template if (settings.Template.IsListTemplate) { url = TestableGlobals.Instance.NavigateURL(modInfo.TabID, ps, "", $"id={itemId}"); } else { // With a signle template we don't want to identify the content by id. url = TestableGlobals.Instance.NavigateURL(modInfo.TabID, ps, ""); } // instanciate the seearch document var retval = new SearchDocument { UniqueKey = modInfo.ModuleID + "-" + itemId + "-" + culture, PortalId = modInfo.PortalID, ModifiedTimeUtc = time, CultureCode = culture, TabId = modInfo.TabID, ModuleId = modInfo.ModuleID, ModuleDefId = modInfo.ModuleDefID, Url = url }; // get the title from the template, if it's there if (!string.IsNullOrEmpty(settings.Template?.Main?.DnnSearchTitle)) { var dicForHbs = JsonUtils.JsonToDictionary(contentData.ToString()); var hbEngine = new HandlebarsEngine(); retval.Title = hbEngine.ExecuteWithoutFaillure(settings.Template.Main.DnnSearchTitle, dicForHbs, modInfo.ModuleTitle); } // SK: this is the behaviour before introduction of DnnSearchTitle else if (dataItemTitle.IsJson()) { if (contentData["Title"] != null) { retval.Title = contentData["Title"].ToString(); } else { retval.Title = modInfo.ModuleTitle; } } else { retval.Title = dataItemTitle; } // for the search text, we're using the template in DnnSearchText if it's used // otherwise, we fall back to previous behaviour: // - if the item has a field called Description, we use that // - otherwise just get the whole item contents if (!string.IsNullOrEmpty(settings.Template?.Main?.DnnSearchText)) { var dicForHbs = JsonUtils.JsonToDictionary(contentData.ToString()); var hbEngine = new HandlebarsEngine(); retval.Body = hbEngine.ExecuteWithoutFaillure(settings.Template.Main.DnnSearchText, dicForHbs, modInfo.ModuleTitle); } else if (contentData["Description"] != null) { retval.Body = contentData["Description"]?.ToString(); } else { retval.Body = JsonToSearchableString(contentData); } // for description, we also try and use the available template first // if that's not there, we'll use the body text for the search document if (!string.IsNullOrEmpty(settings.Template?.Main?.DnnSearchDescription)) { var dicForHbs = JsonUtils.JsonToDictionary(contentData.ToString()); var hbEngine = new HandlebarsEngine(); retval.Description = hbEngine.ExecuteWithoutFaillure(settings.Template.Main.DnnSearchDescription, dicForHbs, modInfo.ModuleTitle); } else { retval.Description = retval.Body; } retval.Title = HttpUtility.HtmlDecode(retval.Title).StripHtml(); retval.Body = HttpUtility.HtmlDecode(retval.Body).StripHtml(); retval.Description = HttpUtility.HtmlDecode(retval.Description).StripHtml(); // Add support for module terms if (modInfo.Terms != null && modInfo.Terms.Count > 0) { retval.Tags = CollectHierarchicalTags(modInfo.Terms); } return(retval); }
public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo modInfo, DateTime beginDateUtc) { Log.Logger.TraceFormat("Indexing content Module {0} - Tab {1} - indexing from {3}", modInfo.ModuleID, modInfo.TabID, modInfo.CultureCode, beginDateUtc); var searchDocuments = new List <SearchDocument>(); //If module is marked as "don't index" then return no results if (modInfo.ModuleSettings.GetValue("AllowIndex", "True") == "False") { Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - MODULE Indexing disabled", modInfo.ModuleID, modInfo.CultureCode); return(searchDocuments); } //If tab of the module is marked as "don't index" then return no results if (modInfo.ParentTab.TabSettings.GetValue("AllowIndex", "True") == "False") { Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - TAB Indexing disabled", modInfo.ModuleID, modInfo.CultureCode); return(searchDocuments); } //If tab is marked as "inactive" then return no results if (modInfo.ParentTab.DisableLink) { Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - TAB is inactive", modInfo.ModuleID, modInfo.CultureCode); return(searchDocuments); } var module = new OpenContentModuleInfo(modInfo); OpenContentSettings settings = modInfo.OpenContentSettings(); //if (settings.Template?.Main == null || !settings.Template.Main.DnnSearch) //{ // Log.Logger.TraceFormat("Indexing content Module {0} - Tab {1} - settings.Template?.Main == null || !settings.Template.Main.DnnSearch", modInfo.ModuleID, modInfo.TabID, modInfo.CultureCode); // return searchDocuments; //} if (settings.IsOtherModule) { return(searchDocuments); } IDataSource ds = DataSourceManager.GetDataSource(module.Settings.Manifest.DataSource); var dsContext = OpenContentUtils.CreateDataContext(module); IDataItems contentList = ds.GetAll(dsContext, null); if (!contentList.Items.Any()) { Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - No content found", modInfo.ModuleID, modInfo.CultureCode); } foreach (IDataItem content in contentList.Items) { if (content == null) { Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - Content is Null", modInfo.ModuleID, modInfo.CultureCode); } else if (content.LastModifiedOnDate.ToUniversalTime() > beginDateUtc && content.LastModifiedOnDate.ToUniversalTime() < DateTime.UtcNow) { SearchDocument searchDoc; if (DnnLanguageUtils.IsMultiLingualPortal(modInfo.PortalID)) { searchDoc = GetLocalizedItem(modInfo, settings, content); searchDocuments.Add(searchDoc); if (modInfo.LocalizedModules != null) { foreach (var localizedModule in modInfo.LocalizedModules) { SearchDocument localizedSearchDoc = GetLocalizedItem(localizedModule.Value, settings, content); searchDocuments.Add(localizedSearchDoc); } } } else { searchDoc = CreateSearchDocument(modInfo, settings, content.Data, content.Id, "", content.Title, JsonToSearchableString(content.Data), content.LastModifiedOnDate.ToUniversalTime()); searchDocuments.Add(searchDoc); Log.Logger.TraceFormat("Indexing content {0}|{5} - OK! {1} ({2}) of {3}", modInfo.ModuleID, searchDoc.Title, modInfo.TabID, content.LastModifiedOnDate.ToUniversalTime(), modInfo.CultureCode); } } else { Log.Logger.TraceFormat("Indexing content {0}|{1} - NOT - No need to index: lastmod {2} ", modInfo.ModuleID, modInfo.CultureCode, content.LastModifiedOnDate.ToUniversalTime()); } } return(searchDocuments); }
/// <inheritdoc /> public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo moduleInfo, DateTime beginDateUtc) { if (moduleInfo == null) { throw new ArgumentNullException("modInfo", @"modInfo must not be null."); } var searchItems = new List <SearchDocument>(); // only index the JobDetail module definition (since most of this information is only viewable there, // and because the Guid parameter on the SearchItemInfo ("jobid=" + jobid) gets put on the querystring to make it work all automagically). BD if (ModuleDefinitionController.GetModuleDefinitionByFriendlyName( ModuleDefinition.JobDetail.ToString(), moduleInfo.DesktopModuleID) .ModuleDefID != moduleInfo.ModuleDefID || !ModuleSettings.JobDetailEnableDnnSearch.GetValueAsBooleanFor( DesktopModuleName, moduleInfo, ModuleSettings.JobDetailEnableDnnSearch.DefaultValue)) { return(searchItems); } int?jobGroupId = ModuleSettings.JobGroupId.GetValueAsInt32For( DesktopModuleName, moduleInfo, ModuleSettings.JobGroupId.DefaultValue); // TODO: only retrieve jobs modified after beginDateUtc using (IDataReader jobs = DataProvider.Instance() .GetJobs(jobGroupId, moduleInfo.PortalID)) { while (jobs.Read()) { var jobId = ((int)jobs["JobId"]).ToString(CultureInfo.InvariantCulture); var revisionDate = (DateTime)jobs["RevisionDate"]; var searchDocument = new SearchDocument { UniqueKey = "Employment_Job_" + jobId, ModifiedTimeUtc = revisionDate.ToUniversalTime(), ModuleId = moduleInfo.ModuleID, PortalId = moduleInfo.PortalID, QueryString = "jobid=" + jobId, }; searchItems.Add(searchDocument); if ((bool)jobs["IsFilled"]) { searchDocument.IsActive = false; continue; } var searchDescription = HtmlUtils.StripWhiteSpace( HtmlUtils.Clean((string)jobs["JobDescription"], false), true); var searchItemTitle = string.Format( CultureInfo.CurrentCulture, Utility.GetString("JobInLocation", LocalResourceFile, moduleInfo.PortalID), (string)jobs["JobTitle"], (string)jobs["LocationName"], (string)jobs["StateName"]); var searchedContent = HtmlUtils.StripWhiteSpace( HtmlUtils.Clean( (string)jobs["JobTitle"] + " " + (string)jobs["JobDescription"] + " " + (string)jobs["RequiredQualifications"] + " " + (string)jobs["DesiredQualifications"], false), true); searchDocument.Title = searchItemTitle; searchDocument.Description = searchDescription; searchDocument.AuthorUserId = (int)jobs["RevisingUser"]; searchDocument.Body = searchedContent; } } return(searchItems); }
public static bool IsVisible(this SearchDocument doc, Guid?userIdenfitier) { return(null != doc && (null == doc.UserIdentifiers || doc.UserIdentifiers.Any(id => id == userIdenfitier))); }
public void AddSearchDocument(SearchDocument searchDocument) { this.AddSearchDocumentInternal(searchDocument, false); }
public static Document ToDocument(this SearchDocument content) { var doc = new Document(); doc.Add(new Field(SearchDocument.IdentifierKey, content.Identifier.ToString(), Field.Store.YES, Field.Index.NO)); doc.Add(new Field(SearchDocument.UserIdentifierKey, content.UserIdentifier.ToString(), Field.Store.YES, Field.Index.ANALYZED)); doc.Add(new Field(SearchDocument.MemberNameKey, content.MemberName, Field.Store.YES, Field.Index.ANALYZED)); var typeField = new NumericField(SearchDocument.TypeKey, Field.Store.YES, true); typeField.SetIntValue((int)content.Type); doc.Add(typeField); var latField = new NumericField(SearchDocument.LatitudeKey, Field.Store.YES, true); latField.SetDoubleValue(content.Latitude); doc.Add(latField); var longField = new NumericField(SearchDocument.LongitudeKey, Field.Store.YES, true); longField.SetDoubleValue(content.Longitude); doc.Add(longField); doc.Add(new Field(SearchDocument.CreatedOnKey, content.CreatedOn.ToStringExact(), Field.Store.YES, Field.Index.NO)); var permissions = content.Permissions(); if (!string.IsNullOrWhiteSpace(permissions)) { doc.Add(new Field(SearchDocument.PermissionsKey, permissions, Field.Store.YES, Field.Index.NO)); } if (!string.IsNullOrWhiteSpace(content.ImageData)) { doc.Add(new Field(SearchDocument.ImageDataKey, content.ImageData, Field.Store.YES, Field.Index.NO)); } if (!string.IsNullOrWhiteSpace(content.Title)) { doc.Add(new Field(SearchDocument.TitleKey, content.Title, Field.Store.YES, Field.Index.NO)); } if (!string.IsNullOrWhiteSpace(content.Description)) { doc.Add(new Field(SearchDocument.DescriptionKey, content.Description, Field.Store.YES, Field.Index.NO)); } if (!string.IsNullOrWhiteSpace(content.Key)) { doc.Add(new Field(SearchDocument.KeyKey, content.Key, Field.Store.YES, Field.Index.NOT_ANALYZED)); } if (!string.IsNullOrWhiteSpace(content.Content)) { doc.Add(new Field(SearchDocument.ContentKey, content.Content, Field.Store.YES, Field.Index.ANALYZED)); } if (!string.IsNullOrWhiteSpace(content.Location)) { doc.Add(new Field(SearchDocument.LocationKey, content.Location, Field.Store.YES, Field.Index.ANALYZED)); } return(doc); }
/// ----------------------------------------------------------------------------- /// <summary> /// Returns the collection of SearchDocuments populated with Tab MetaData for the given portal. /// </summary> /// <param name="portalId"></param> /// <param name="startDateLocal"></param> /// <returns></returns> /// <history> /// [vnguyen] 04/16/2013 created /// </history> /// ----------------------------------------------------------------------------- public override IEnumerable <SearchDocument> GetSearchDocuments(int portalId, DateTime startDateLocal) { var searchDocuments = new Dictionary <string, SearchDocument>(); var needReindex = PortalController.GetPortalSettingAsBoolean(UserIndexResetFlag, portalId, false); if (needReindex) { startDateLocal = SqlDateTime.MinValue.Value.AddDays(1); } var controller = new ListController(); var textDataType = controller.GetListEntryInfo("DataType", "Text"); var richTextDataType = controller.GetListEntryInfo("DataType", "RichText"); var profileDefinitions = ProfileController.GetPropertyDefinitionsByPortal(portalId, false, false) .Cast <ProfilePropertyDefinition>() .Where(d => (textDataType != null && d.DataType == textDataType.EntryID) || (richTextDataType != null && d.DataType == richTextDataType.EntryID)).ToList(); try { var startUserId = Null.NullInteger; while (true) { var reader = DataProvider.Instance() .GetAvailableUsersForIndex(portalId, startDateLocal, startUserId, BatchSize); int rowsAffected = 0; var indexedUsers = new List <int>(); while (reader.Read()) { var userSearch = GetUserSearch(reader); AddBasicInformation(searchDocuments, indexedUsers, userSearch, portalId); //log the userid so that it can get the correct user collection next time. if (userSearch.UserId > startUserId) { startUserId = userSearch.UserId; } foreach (var definition in profileDefinitions) { var propertyName = definition.PropertyName; if (!ContainsColumn(propertyName, reader)) { continue; } var propertyValue = reader[propertyName].ToString(); if (string.IsNullOrEmpty(propertyValue) || !propertyValue.Contains(ValueSplitFlag)) { continue; } var splitValues = Regex.Split(propertyValue, Regex.Escape(ValueSplitFlag)); propertyValue = splitValues[0]; var visibilityMode = ((UserVisibilityMode)Convert.ToInt32(splitValues[1])); var extendedVisibility = splitValues[2]; var modifiedTime = Convert.ToDateTime(splitValues[3]).ToUniversalTime(); if (string.IsNullOrEmpty(propertyValue)) { continue; } var uniqueKey = string.Format("{0}_{1}", userSearch.UserId, visibilityMode).ToLowerInvariant(); if (visibilityMode == UserVisibilityMode.FriendsAndGroups) { uniqueKey = string.Format("{0}_{1}", uniqueKey, extendedVisibility); } if (searchDocuments.ContainsKey(uniqueKey)) { var document = searchDocuments[uniqueKey]; document.Keywords.Add(propertyName, propertyValue); if (modifiedTime > document.ModifiedTimeUtc) { document.ModifiedTimeUtc = modifiedTime; } } else { //Need remove use exists index for all visibilities. if (!indexedUsers.Contains(userSearch.UserId)) { indexedUsers.Add(userSearch.UserId); } if (!string.IsNullOrEmpty(propertyValue)) { var searchDoc = new SearchDocument { SearchTypeId = UserSearchTypeId, UniqueKey = uniqueKey, PortalId = portalId, ModifiedTimeUtc = modifiedTime, Description = userSearch.FirstName, Title = userSearch.DisplayName }; searchDoc.Keywords.Add(propertyName, propertyValue); searchDoc.NumericKeys.Add("superuser", Convert.ToInt32(userSearch.SuperUser)); searchDocuments.Add(uniqueKey, searchDoc); } } } rowsAffected++; } //close the data reader reader.Close(); //remov exists indexes DeleteDocuments(portalId, indexedUsers); if (rowsAffected == 0) { break; } } if (needReindex) { PortalController.DeletePortalSetting(portalId, UserIndexResetFlag); } } catch (Exception ex) { Exceptions.Exceptions.LogException(ex); } return(searchDocuments.Values); }
private ISearchResults Search <T>(ISearchCriteria criteria) where T : EntryContentBase { // set up filters // 1: phrase var filters = new List <Nest.QueryContainer>(); if (criteria is CatalogEntrySearchCriteria) { var cesc = criteria as CatalogEntrySearchCriteria; if (!string.IsNullOrWhiteSpace(cesc.SearchPhrase)) { filters.Add(new Nest.QueryContainerDescriptor <T>().SimpleQueryString( sq => sq.Fields(f => f.Field("*.analyzed")).Query(cesc.SearchPhrase.Trim()))); } } // 2: id ... NOTE: Vulcan supports 1 and only 1 filter field, code if (criteria.ActiveFilterFields != null && criteria.ActiveFilterFields.Count() == 1 && criteria.ActiveFilterFields[0].Equals("code", StringComparison.InvariantCultureIgnoreCase)) { filters.Add(new Nest.QueryContainerDescriptor <T>().Term( p => p.Field(f => f.Code).Value((criteria.ActiveFilterValues[0] as SimpleValue).value))); } // 3: inactive... TODO, not sure what this should check! /* * if(!criteria.IncludeInactive) * { * filters.Add(new Nest.QueryContainerDescriptor<T>().Term( * p => p.Field(f => f.) * }*/ // get catalog filter, if needed var catalogReferences = new List <ContentReference>(); if (criteria is CatalogEntrySearchCriteria) { var cesc = criteria as CatalogEntrySearchCriteria; if (cesc.CatalogNames != null) { var catalogs = ContentLoader.Service.GetChildren <CatalogContent>(ReferenceConverter.Service.GetRootLink()); if (catalogs != null && catalogs.Any()) { foreach (var catalogName in cesc.CatalogNames) { var catalog = catalogs.FirstOrDefault(c => c.Name.Equals(catalogName, StringComparison.InvariantCultureIgnoreCase)); if (catalog != null) { catalogReferences.Add(catalog.ContentLink); } } } } } if (!catalogReferences.Any()) { catalogReferences = null; } // do search var searchDescriptor = new Nest.SearchDescriptor <T>(); searchDescriptor.Skip(criteria.StartingRecord); searchDescriptor.Take(criteria.RecordsToRetrieve); if (filters.Any()) { searchDescriptor.Query(q => q.Bool(b => b.Must(filters.ToArray()))); } var client = VulcanHandler.Service.GetClient(new CultureInfo(criteria.Locale)); var results = client.SearchContent <T>(q => searchDescriptor, false, catalogReferences); //var id = ReferenceConverter.Service.GetObjectId(); var searchDocuments = new SearchDocuments() { TotalCount = Convert.ToInt32(results.Total) }; if (results.Hits != null && results.Hits.Any()) { foreach (var hit in results.Hits) { var doc = new SearchDocument(); doc.Add(new SearchField("_id", ReferenceConverter.Service.GetObjectId(hit.Source.ContentLink))); searchDocuments.Add(doc); } } return(new SearchResults(searchDocuments, criteria)); }
public bool ReadDocuments(string dataFilePath) { List<SearchDocument> tmpDocuments = new List<SearchDocument>(); StringBuilder sb = new StringBuilder(); string header = string.Empty; try { using (FileStream fs = new FileStream(dataFilePath, FileMode.Open, FileAccess.Read)) { using (StreamReader sr = new StreamReader(fs)) { while (!sr.EndOfStream) { sb.Remove(0, sb.Length); string line; int counter = 0; header = string.Empty; while(((line = sr.ReadLine()) != string.Empty)) { if (counter == 0) header = string.Format("{0} ", line); else sb.Append(string.Format("{0} ", line)); counter++; if (sr.EndOfStream) break; } SearchDocument document = new SearchDocument(header, sb.ToString()); tmpDocuments.Add(document); } } } } catch (FileNotFoundException ex) { throw new ArgumentException(String.Format("Plik nie został znaleziony. {0}", ex.Message)); } catch (IOException ex) { throw new ArgumentException(String.Format("Błąd podczas odczytu pliku", ex.Message)); } // czyszczenie i stemming wczytanych dokumentów if (tmpDocuments.Count > 0) { for(int i=0; i < tmpDocuments.Count; i++) tmpDocuments[i].ProcessElement(textProcessor); documents = tmpDocuments; return true; } else return false; }
public void ModifyingDocumentsDoesNotCreateDuplicates() { // Arrange const string tabUrl = "mysite/ContentUrl"; const string title = "content title"; const string contentBody = "content body"; const string titleModified = title + " modified"; var uniqueKey = Guid.NewGuid().ToString(); var now = DateTime.UtcNow; var originalDocument = new SearchDocument { UniqueKey = uniqueKey, TabId = TabId1, Url = tabUrl, Title = title, Body = contentBody, SearchTypeId = TabSearchTypeId, ModifiedTimeUtc = now, PortalId = PortalId0, RoleId = RoleId731, Keywords = { { "description", "mycontent" } }, NumericKeys = { { "points", 5 } }, }; this.internalSearchController.AddSearchDocument(originalDocument); this.internalSearchController.Commit(); var modifiedDocument = new SearchDocument { UniqueKey = uniqueKey, TabId = TabId1, Url = tabUrl, Title = titleModified, Body = contentBody + " modified", SearchTypeId = TabSearchTypeId, ModifiedTimeUtc = now, PortalId = PortalId0, RoleId = RoleId731, Keywords = { { "description", "mycontent_modified" }, { "description2", "mycontent_modified" } }, NumericKeys = { { "points", 8 }, { "point2", 7 } }, }; this.internalSearchController.AddSearchDocument(modifiedDocument); this.internalSearchController.Commit(); var query = new SearchQuery { KeyWords = title, PortalIds = new List <int> { PortalId0 }, SearchTypeIds = new[] { ModuleSearchTypeId, TabSearchTypeId, UserSearchTypeId }, BeginModifiedTimeUtc = now.AddMinutes(-1), EndModifiedTimeUtc = now.AddMinutes(+1), PageIndex = 1, PageSize = 15, SortField = 0, TitleSnippetLength = 120, BodySnippetLength = 300, WildCardSearch = true, }; // Run var searchResults = this.GetGroupedDetailViewResults(query).ToList(); // Assert Assert.AreEqual(1, searchResults.Count()); Assert.AreEqual(1, searchResults.First().Results.Count); Assert.AreEqual(tabUrl, searchResults.First().Results.First().DocumentUrl); Assert.AreEqual(titleModified, searchResults.First().Results.First().Title); }
/// ----------------------------------------------------------------------------- /// <summary> /// Implements the search interface for DotNetNuke /// </summary> /// ----------------------------------------------------------------------------- public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo modInfo, DateTime beginDateUtc) { var searchDocuments = new List <SearchDocument>(); var udtController = new UserDefinedTableController(modInfo); try { var dsUserDefinedRows = udtController.GetDataSet(withPreRenderedValues: false); //Get names of ChangedBy and ChangedAt columns var colnameChangedBy = udtController.ColumnNameByDataType(dsUserDefinedRows, DataTypeNames.UDT_DataType_ChangedBy); var colnameChangedAt = udtController.ColumnNameByDataType(dsUserDefinedRows, DataTypeNames.UDT_DataType_ChangedAt); var settings = modInfo.ModuleSettings; var includeInSearch = !(settings[SettingName.ExcludeFromSearch].AsBoolean()); if (includeInSearch) { foreach (DataRow row in dsUserDefinedRows.Tables[DataSetTableName.Data].Rows) { var changedDate = DateTime.Today; var changedByUserId = 0; if (colnameChangedAt != string.Empty && !Information.IsDBNull(row[colnameChangedAt])) { changedDate = Convert.ToDateTime(row[colnameChangedAt]); } if (colnameChangedBy != string.Empty && !Information.IsDBNull(row[colnameChangedBy])) { changedByUserId = ModuleSecurity.UserId(row[colnameChangedBy].ToString(), modInfo.PortalID); } var desc = string.Empty; foreach (DataRow col in dsUserDefinedRows.Tables[DataSetTableName.Fields].Rows) { var fieldType = col[FieldsTableColumn.Type].ToString(); var fieldTitle = col[FieldsTableColumn.Title].ToString(); var visible = Convert.ToBoolean(col[FieldsTableColumn.Visible]); if (visible && (fieldType.StartsWith("Text") || fieldType == DataTypeNames.UDT_DataType_String)) { desc += string.Format("{0} • ", Convert.ToString(row[fieldTitle])); } } if (desc.EndsWith("<br/>")) { desc = desc.Substring(0, Convert.ToInt32(desc.Length - 5)); } var searchDoc = new SearchDocument { UniqueKey = row[DataTableColumn.RowId].ToString(), PortalId = modInfo.PortalID, Title = modInfo.ModuleTitle, Description = desc, Body = desc, ModifiedTimeUtc = changedDate }; searchDocuments.Add(searchDoc); } } } catch (Exception ex) { Exceptions.LogException(ex); } return(searchDocuments); }
public virtual void IncludeDefaultField(Document document, IContent content) { var versionable = (content as IVersionable); var changeTrackable = content as IChangeTrackable; var contentStatus = ((int)versionable.Status).ToString(); var virtualPath = ContentIndexHelpers.GetContentVirtualpath(content); var acl = ContentIndexHelpers.GetContentACL(content); var contentType = ContentIndexHelpers.GetContentType(content); var language = (content as ILocalizable).Language.Name; var startPublishDate = versionable.StartPublish.HasValue ? DateTools.DateToString(versionable.StartPublish.Value.ToUniversalTime(), DateTools.Resolution.SECOND) : string.Empty; var stopPublishDate = versionable.StopPublish.HasValue ? DateTools.DateToString(versionable.StopPublish.Value.ToUniversalTime(), DateTools.Resolution.SECOND) : string.Empty; var expired = ""; if ((versionable.StopPublish.HasValue && versionable.StopPublish.Value.ToUniversalTime() < DateTime.UtcNow) || (versionable.StartPublish.HasValue && versionable.StartPublish.Value.ToUniversalTime() > DateTime.UtcNow)) { expired = "true"; } var createdDate = DateTools.DateToString(changeTrackable.Created.ToUniversalTime(), DateTools.Resolution.SECOND); var updatedDate = DateTools.DateToString(changeTrackable.Changed.ToUniversalTime(), DateTools.Resolution.SECOND); var idField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_ID), SearchDocument.FormatDocumentId(content), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var nameField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_NAME), content.Name, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var statusField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_STATUS), contentStatus, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var virtualPathField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_VIRTUAL_PATH), virtualPath, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var aclField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_ACL), acl, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var typeField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_TYPE), contentType, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var langField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_LANGUAGE), language, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var startPublishField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_START_PUBLISHDATE), startPublishDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var stopPublishField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_STOP_PUBLISHDATE), stopPublishDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var referenceField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_REFERENCE), content.ContentLink.ID.ToString(), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var createdField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_CREATED), createdDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var updatedField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_CHANGED), updatedDate, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); var expiredField = new Field(ContentIndexHelpers.GetIndexFieldName(Constants.INDEX_FIELD_NAME_EXPIRED), expired, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.YES); document.Add(idField); document.Add(nameField); document.Add(statusField); document.Add(virtualPathField); document.Add(aclField); document.Add(typeField); document.Add(langField); document.Add(startPublishField); document.Add(stopPublishField); document.Add(referenceField); document.Add(createdField); document.Add(updatedField); document.Add(expiredField); }
public async Task <IActionResult> GetSearchDocument([FromQuery] SearchDocument SearchDocument) //int catId, int subcatId, int officeId, int companyId, string date) { //var items = await _context.Document.Where(x => EF.Functions.FreeText(x.DescrizioneDocumento, SearchDocument.SearchText)).ToListAsync(); if (SearchDocument.SearchText != null) { SearchDocument.SearchText = SearchDocument.SearchText.TrimEnd(); SearchDocument.SearchText = string.Join(" ", SearchDocument.SearchText.Split(' ').Select(w => string.Format("+{0}*", w))); } if (!ModelState.IsValid) { return(BadRequest(ModelState)); } SearchDocument.EndDate = SearchDocument.EndDate.Date.AddHours(23).AddMinutes(59).AddSeconds(59); List <Document> document; int totalCount; if (SearchDocument.SearchText != null) { document = await _context.Document.FromSql("CALL FullTextSearch({0})", SearchDocument.SearchText).ToListAsync(); } else { document = await _context.Document.ToListAsync(); } if (SearchDocument.CategoriaId != 0 || SearchDocument.SottocategoriaId != 0 || SearchDocument.SocietaId != 0 || SearchDocument.UfficioId != 0 || SearchDocument.StartDate != new DateTime(0001, 1, 1) || SearchDocument.Parte2Id != 0 || SearchDocument.Parte3Id != 0) { if (SearchDocument.CategoriaId != 0) { document = document.Where(x => x.CategoriaId == SearchDocument.CategoriaId).ToList(); } if (SearchDocument.SottocategoriaId != 0) { document = document.Where(x => x.SottocategoriaId == SearchDocument.SottocategoriaId).ToList(); } if (SearchDocument.SocietaId != 0) { document = document.Where(x => x.SocietaId == SearchDocument.SocietaId).ToList(); } if (SearchDocument.UfficioId != 0) { document = document.Where(x => x.UfficioId == SearchDocument.UfficioId).ToList(); } if (SearchDocument.Parte2Id != 0) { document = document.Where(x => x.Parte2Id == SearchDocument.Parte2Id).ToList(); } if (SearchDocument.Parte3Id != 0) { document = document.Where(x => x.Parte3Id == SearchDocument.Parte3Id).ToList(); } if (SearchDocument.StartDate != new DateTime(0001, 1, 1)) { document = document.Where(x => x.DataDocumento >= SearchDocument.StartDate.Date && x.DataDocumento <= SearchDocument.EndDate).ToList(); } } int p = (SearchDocument.CurrentPage * 20) - 20; totalCount = document.Count(); if (SearchDocument.SortColumn != null && SearchDocument.SortDirection != null) { switch (SearchDocument.SortColumn) { case "dataScadenza": if (SearchDocument.SortDirection == "asc") { document = document.OrderBy(x => x.DataScadenza).Skip(p).Take(20).ToList(); } else { document = document.OrderByDescending(x => x.DataScadenza).Skip(p).Take(20).ToList(); } break; case "dataDocumento": if (SearchDocument.SortDirection == "asc") { document = document.OrderBy(x => x.DataDocumento).Skip(p).Take(20).ToList(); } else { document = document.OrderByDescending(x => x.DataDocumento).Skip(p).Take(20).ToList(); } break; default: document = document.OrderByDescending(x => x.DataDocumento).Skip(p).Take(20).ToList(); break; } } else { document = document.OrderByDescending(x => x.DataDocumento).Skip(p).Take(20).ToList(); } Response.Headers.Add("X-Pagination", JsonConvert.SerializeObject(totalCount)); return(Ok(document)); }
public Result GetResults(Query searchQuery) { Uri searchEndpoint = new Uri(_configuration["SearchServiceEndpoint"]); string apiKey = _configuration["SearchServiceQueryApiKey"]; string indexName = _configuration["SearchIndexName"]; Result results = new Result(); List <Dictionary <string, string> > processedResults = new List <Dictionary <string, string> >(); Dictionary <string, List <FacetResult> > facetResults = new Dictionary <string, List <FacetResult> >(); Dictionary <string, string> speakersDict = _configuration.GetSection("Speakers").GetChildren().ToDictionary(x => x.Key, x => x.Value); SearchOptions options = new SearchOptions { Size = 100, IncludeTotalCount = true, Facets = { "duration,sort:count", "recognizedPhrases/speaker,sort:count" }, HighlightFields = { "combinedRecognizedPhrases/display" }, HighlightPreTag = "<span class='hitHighlight'>", HighlightPostTag = "</span>", Filter = searchQuery.filter, SearchMode = SearchMode.All }; AzureKeyCredential credential = new AzureKeyCredential(apiKey); SearchClient client = new SearchClient(searchEndpoint, indexName, credential); SearchResults <SearchDocument> response = client.Search <SearchDocument>(searchQuery.searchText, options); foreach (SearchResult <SearchDocument> result in response.GetResults()) { SearchDocument doc = result.Document; Dictionary <string, string> r = new Dictionary <string, string>(); string speakerTranscription = string.Empty; r.Add("filename", (string)doc["metadata_storage_name"]); r.Add("timestamp", (string)doc["timeStamp"]); //results.processedResults.Add("size", (string)doc["metadata_storage_size"]); r.Add("lastmodified", (string)doc["metadata_storage_last_modified"]); r.Add("transcription", (string)doc.GetObjectCollection("combinedRecognizedPhrases")[0]["display"]); foreach (SearchDocument recognizedPhrase in doc.GetObjectCollection("recognizedPhrases")) { speakerTranscription += String.Format("<span class=\"speakerLabel\">Speaker {0}</span>: {1} <br>", recognizedPhrase.GetInt32("speaker").ToString(), recognizedPhrase.GetObjectCollection("nBest")[0].GetString("display")); } r.Add("speakerTranscription", speakerTranscription); if (result.Highlights != null) { r.Add("highlights", String.Join(" ... ", result.Highlights["combinedRecognizedPhrases/display"].ToList())); } else { r.Add("highlights", r["transcription"].Substring(0, Math.Min(r["transcription"].Length, 200)) + "..."); } r.Add("source", (string)doc["source"]); processedResults.Add(r); } foreach (var f in response.Facets) { switch (f.Key) { case "recognizedPhrases/speaker": facetResults["recognizedPhrases/speaker"] = f.Value.ToList(); continue; case "duration": facetResults["duration"] = f.Value.ToList(); continue; } } results.processedResults = processedResults; results.facets = facetResults; results.count = response.TotalCount; return(results); }
public override IList <SearchDocument> GetModifiedSearchDocuments(ModuleInfo moduleInfo, DateTime beginDate) { var searchDocuments = new Dictionary <string, SearchDocument>(); int lastJournalId = Null.NullInteger; if (beginDate == DateTime.MinValue) { beginDate = SqlDateTime.MinValue.Value; } if (beginDate > SqlDateTime.MinValue.Value) { beginDate = beginDate.ToUniversalTime(); } try { while (true) { var reader = DataProvider.Instance() .ExecuteReader("Journal_GetSearchItems", moduleInfo.PortalID, moduleInfo.TabModuleID, beginDate, lastJournalId, Constants.SearchBatchSize); var journalIds = new Dictionary <int, int>(); while (reader.Read()) { var journalId = Convert.ToInt32(reader["JournalId"]); var journalTypeId = reader["JournalTypeId"].ToString(); var userId = Convert.ToInt32(reader["UserId"]); var dateUpdated = Convert.ToDateTime(reader["DateUpdated"]); var profileId = reader["ProfileId"].ToString(); var groupId = reader["GroupId"].ToString(); var title = reader["Title"].ToString(); var summary = reader["Summary"].ToString(); var securityKey = reader["SecurityKey"].ToString(); var tabId = reader["TabId"].ToString(); var tabModuleId = reader["ModuleId"].ToString(); var key = string.Format("JI_{0}", journalId); if (searchDocuments.ContainsKey(key)) { searchDocuments[key].UniqueKey += string.Format(",{0}", securityKey); } else { var searchDocument = new SearchDocument() { UniqueKey = string.Format("JI_{0}_{1}", journalId, securityKey), Body = summary, ModifiedTimeUtc = dateUpdated, Title = title, AuthorUserId = userId, Keywords = new Dictionary <string, string>() { { "TabId", tabId }, { "TabModuleId", tabModuleId }, { "ProfileId", profileId }, { "GroupId", groupId } } }; searchDocuments.Add(key, searchDocument); } if (journalId > lastJournalId) { lastJournalId = journalId; } if (!journalIds.ContainsKey(journalId)) { journalIds.Add(journalId, userId); } } //close the reader reader.Close(); if (journalIds.Count == 0) { break; } else { //index comments for this journal AddCommentItems(journalIds, searchDocuments); } } } catch (Exception ex) { Exceptions.LogException(ex); } return(searchDocuments.Values.ToList()); }
private static int FindModifiedUsers(int portalId, DateTime startDateLocal, IDictionary <string, SearchDocument> searchDocuments, IList <ProfilePropertyDefinition> profileDefinitions, out IList <int> indexedUsers, ref int startUserId) { var rowsAffected = 0; indexedUsers = new List <int>(); using (var reader = DataProvider.Instance().GetAvailableUsersForIndex(portalId, startDateLocal, startUserId, BatchSize)) { while (reader.Read()) { var userSearch = GetUserSearch(reader); if (userSearch == null) { continue; } AddBasicInformation(searchDocuments, indexedUsers, userSearch, portalId); //log the userid so that it can get the correct user collection next time. if (userSearch.UserId > startUserId) { startUserId = userSearch.UserId; } foreach (var definition in profileDefinitions) { var propertyName = definition.PropertyName; if (!ContainsColumn(propertyName, reader)) { continue; } var propertyValue = reader[propertyName].ToString(); if (string.IsNullOrEmpty(propertyValue) || !propertyValue.Contains(ValueSplitFlag)) { continue; } var splitValues = Regex.Split(propertyValue, Regex.Escape(ValueSplitFlag)); propertyValue = splitValues[0]; var visibilityMode = ((UserVisibilityMode)Convert.ToInt32(splitValues[1])); var extendedVisibility = splitValues[2]; var modifiedTime = Convert.ToDateTime(splitValues[3]).ToUniversalTime(); if (string.IsNullOrEmpty(propertyValue)) { continue; } //DNN-5740: replace split flag if it included in property value. propertyValue = propertyValue.Replace("[$][$][$]", "$$$"); var uniqueKey = string.Format("{0}_{1}", userSearch.UserId, visibilityMode).ToLowerInvariant(); if (visibilityMode == UserVisibilityMode.FriendsAndGroups) { uniqueKey = string.Format("{0}_{1}", uniqueKey, extendedVisibility); } if (searchDocuments.ContainsKey(uniqueKey)) { var document = searchDocuments[uniqueKey]; document.Keywords.Add(propertyName, propertyValue); if (modifiedTime > document.ModifiedTimeUtc) { document.ModifiedTimeUtc = modifiedTime; } } else { //Need remove use exists index for all visibilities. if (!indexedUsers.Contains(userSearch.UserId)) { indexedUsers.Add(userSearch.UserId); } if (!string.IsNullOrEmpty(propertyValue)) { var searchDoc = new SearchDocument { SearchTypeId = UserSearchTypeId, UniqueKey = uniqueKey, PortalId = portalId, ModifiedTimeUtc = modifiedTime, Description = userSearch.FirstName, Title = userSearch.DisplayName }; searchDoc.Keywords.Add(propertyName, propertyValue); searchDoc.NumericKeys.Add("superuser", Convert.ToInt32(userSearch.SuperUser)); searchDocuments.Add(uniqueKey, searchDoc); } } } rowsAffected++; } } return(rowsAffected); }
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddImageUrl() { var entry = GetCatalogEntryRow("Product"); _urlResolverMock.Setup(x => x.GetUrl(new ContentReference(5, 0))) .Returns("http://myimage"); _assetUrlConventionsMock.Setup(x => x.GetDefaultGroup(It.IsAny<IAssetContainer>())) .Returns("default"); var document = new SearchDocument(); _subject.UpdateSearchDocument(ref document, entry, "en"); document["image_url"].Should().Equals("http://myimage"); }
private void AddSearchDocumentInternal(SearchDocument searchDocument, bool autoCommit) { Requires.NotNull("SearchDocument", searchDocument); Requires.NotNullOrEmpty("UniqueKey", searchDocument.UniqueKey); Requires.NotNegative("SearchTypeId", searchDocument.SearchTypeId); Requires.PropertyNotEqualTo("searchDocument", "SearchTypeId", searchDocument.SearchTypeId, 0); Requires.PropertyNotEqualTo("searchDocument", "ModifiedTimeUtc", searchDocument.ModifiedTimeUtc.ToString(CultureInfo.InvariantCulture), DateTime.MinValue.ToString(CultureInfo.InvariantCulture)); if (searchDocument.SearchTypeId == this._moduleSearchTypeId) { if (searchDocument.ModuleDefId <= 0) { throw new ArgumentException(Localization.GetExceptionMessage("ModuleDefIdMustBeGreaterThanZero", "ModuleDefId must be greater than zero when SearchTypeId is for a module")); } if (searchDocument.ModuleId <= 0) { throw new ArgumentException(Localization.GetExceptionMessage("ModuleIdMustBeGreaterThanZero", "ModuleId must be greater than zero when SearchTypeId is for a module")); } } else { if (searchDocument.ModuleDefId > 0) { throw new ArgumentException(Localization.GetExceptionMessage("ModuleDefIdWhenSearchTypeForModule", "ModuleDefId is needed only when SearchTypeId is for a module")); } if (searchDocument.ModuleId > 0) { throw new ArgumentException(Localization.GetExceptionMessage("ModuleIdWhenSearchTypeForModule", "ModuleId is needed only when SearchTypeId is for a module")); } } var doc = new Document(); var sb = new StringBuilder(); // TODO - Set setOmitTermFreqAndPositions // TODO - Check if Numeric fields need to have Store.YES or Store.NO // TODO - Should ModifiedTime be mandatory // TODO - Is Locale needed for a single-language site // TODO - Sorting // Field.Store.YES | Stores the value. When the value is stored, the original String in its entirety is recorded in the index // Field.Store.NO | Doesn’t store the value. This option is often used along with Index.ANALYZED to index a large text field that doesn’t need to be retrieved // | in its original form, such as bodies of web pages, or any other type of text document. // Index.ANALYZED | Use the analyzer to break the field’s value into a stream of separate tokens and make each token searchable // Index.NOT_ANALYZED | Do index the field, but don’t analyze the String value.Instead, treat the Field’s entire value as a single token and make that token searchable. // Generic and Additional SearchDocument Params this.AddSearchDocumentParamters(doc, searchDocument, sb); // Remove the existing document from Lucene this.DeleteSearchDocumentInternal(searchDocument, false); // Add only when Document is active. The previous call would have otherwise deleted the document if it existed earlier if (searchDocument.IsActive) { Thread.SetData(Thread.GetNamedDataSlot(Constants.TlsSearchInfo), searchDocument); try { LuceneController.Instance.Add(doc); } finally { Thread.SetData(Thread.GetNamedDataSlot(Constants.TlsSearchInfo), null); } } if (autoCommit) { this.Commit(); } }
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddCreated() { var entry = GetCatalogEntryRow("Product"); var document = new SearchDocument(); _subject.UpdateSearchDocument(ref document, entry, "en"); document["created"].Should().Equals(new DateTime(2012, 4, 4).ToString("yyyyMMddhhmmss")); }
private void AddSearchDocumentParamters(Document doc, SearchDocument searchDocument, StringBuilder sb) { // mandatory fields doc.Add(new Field(Constants.UniqueKeyTag, SearchHelper.Instance.StripTagsNoAttributes(searchDocument.UniqueKey, true), Field.Store.YES, Field.Index.NOT_ANALYZED)); doc.Add(new NumericField(Constants.PortalIdTag, Field.Store.YES, true).SetIntValue(searchDocument.PortalId)); doc.Add(new NumericField(Constants.SearchTypeTag, Field.Store.YES, true).SetIntValue(searchDocument.SearchTypeId)); doc.Add(!string.IsNullOrEmpty(searchDocument.CultureCode) ? new NumericField(Constants.LocaleTag, Field.Store.YES, true).SetIntValue(Localization.GetCultureLanguageID(searchDocument.CultureCode)) : new NumericField(Constants.LocaleTag, Field.Store.YES, true).SetIntValue(-1)); if (!string.IsNullOrEmpty(searchDocument.Title)) { var field = new Field(Constants.TitleTag, StripTagsRetainAttributes(searchDocument.Title, HtmlAttributesToRetain, false, true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); if (this._titleBoost > 0 && this._titleBoost != Constants.StandardLuceneBoost) { field.Boost = this._titleBoost / 10f; } doc.Add(field); } if (!string.IsNullOrEmpty(searchDocument.Description)) { var field = new Field(Constants.DescriptionTag, StripTagsRetainAttributes(searchDocument.Description, HtmlAttributesToRetain, false, true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); if (this._descriptionBoost > 0 && this._descriptionBoost != Constants.StandardLuceneBoost) { field.Boost = this._descriptionBoost / 10f; } doc.Add(field); } if (!string.IsNullOrEmpty(searchDocument.Body)) { doc.Add(new Field(Constants.BodyTag, StripTagsRetainAttributes(searchDocument.Body, HtmlAttributesToRetain, false, true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); } if (!string.IsNullOrEmpty(searchDocument.Url)) { doc.Add(new Field(Constants.UrlTag, SearchHelper.Instance.StripTagsNoAttributes(searchDocument.Url, true), Field.Store.YES, Field.Index.NOT_ANALYZED)); } if (!string.IsNullOrEmpty(searchDocument.QueryString)) { doc.Add(new Field(Constants.QueryStringTag, searchDocument.QueryString, Field.Store.YES, Field.Index.NOT_ANALYZED)); } foreach (var kvp in searchDocument.Keywords) { var key = kvp.Key.ToLowerInvariant(); var needAnalyzed = Constants.FieldsNeedAnalysis.Contains(key); var field = new Field(SearchHelper.Instance.StripTagsNoAttributes(Constants.KeywordsPrefixTag + kvp.Key, true), SearchHelper.Instance.StripTagsNoAttributes(kvp.Value, true), Field.Store.YES, needAnalyzed ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED); switch (key) { case Constants.TitleTag: if (this._titleBoost > 0 && this._titleBoost != Constants.StandardLuceneBoost) { field.Boost = this._titleBoost / 10f; } break; case Constants.SubjectTag: if (this._contentBoost > 0 && this._contentBoost != Constants.StandardLuceneBoost) { field.Boost = this._contentBoost / 10f; } break; case Constants.CommentsTag: if (this._descriptionBoost > 0 && this._descriptionBoost != Constants.StandardLuceneBoost) { field.Boost = this._descriptionBoost / 10f; } break; case Constants.AuthorNameTag: if (this._authorBoost > 0 && this._authorBoost != Constants.StandardLuceneBoost) { field.Boost = this._authorBoost / 10f; } break; } doc.Add(field); sb.Append(SearchHelper.Instance.StripTagsNoAttributes(kvp.Value, true)).Append(" "); } foreach (var kvp in searchDocument.NumericKeys) { doc.Add(new NumericField(SearchHelper.Instance.StripTagsNoAttributes(Constants.NumericKeyPrefixTag + kvp.Key, true), Field.Store.YES, true).SetIntValue(kvp.Value)); } bool tagBoostApplied = false; foreach (var tag in searchDocument.Tags) { var field = new Field(Constants.Tag, SearchHelper.Instance.StripTagsNoAttributes(tag.ToLowerInvariant(), true), Field.Store.YES, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); if (!tagBoostApplied) { if (this._tagBoost > 0 && this._tagBoost != Constants.StandardLuceneBoost) { field.Boost = this._tagBoost / 10f; tagBoostApplied = true; } } doc.Add(field); } AddIntField(doc, searchDocument.TabId, Constants.TabIdTag); AddIntField(doc, searchDocument.ModuleDefId, Constants.ModuleDefIdTag); AddIntField(doc, searchDocument.ModuleId, Constants.ModuleIdTag); AddIntField(doc, searchDocument.AuthorUserId, Constants.AuthorIdTag); AddIntField(doc, searchDocument.RoleId, Constants.RoleIdTag); if (searchDocument.AuthorUserId > 0) { var user = UserController.Instance.GetUserById(searchDocument.PortalId, searchDocument.AuthorUserId); if (user != null && !string.IsNullOrEmpty(user.DisplayName)) { var field = new Field(Constants.AuthorNameTag, user.DisplayName, Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); if (this._authorBoost > 0 && this._authorBoost != Constants.StandardLuceneBoost) { field.Boost = this._authorBoost / 10f; } doc.Add(field); } } if (!string.IsNullOrEmpty(searchDocument.Permissions)) { doc.Add(new Field(Constants.PermissionsTag, SearchHelper.Instance.StripTagsNoAttributes(searchDocument.Permissions, true), Field.Store.YES, Field.Index.NOT_ANALYZED)); } doc.Add(new NumericField(Constants.ModifiedTimeTag, Field.Store.YES, true).SetLongValue(long.Parse(searchDocument.ModifiedTimeUtc.ToString(Constants.DateTimeFormat)))); if (sb.Length > 0) { var field = new Field(Constants.ContentTag, SearchHelper.Instance.StripTagsNoAttributes(sb.ToString(), true), Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS); doc.Add(field); if (this._contentBoost > 0 && this._contentBoost != Constants.StandardLuceneBoost) { field.Boost = this._contentBoost / 10f; } } }
public IList <SearchDocument> Search <T>(IList <Filter> searchFilters, IList <SearchSortField> sortFields) where T : class { var connection = new ConnectionSettings(new Uri("http://localhost:9200")); connection.DefaultIndex("search_example"); var filter = CreateFilter(searchFilters); var searchRequest = new SearchRequest <DeveloperModel> { Query = new BoolQuery { Filter = filter }, Size = 100, From = 0 }; var client = new ElasticClient(connection); using (var stream = new MemoryStream()) { client.Serializer.Serialize(searchRequest, stream); string searchJson = Encoding.UTF8.GetString(stream.ToArray()); System.Diagnostics.Trace.WriteLine(searchJson); } var response = client.Search <DeveloperModel>(searchRequest); var result = new List <SearchDocument>(); if (response.IsValid) { foreach (var model in response.Documents) { var doc = new SearchDocument { Id = model.Id, Fields = new List <SearchDocumentField> { new SearchDocumentField { FieldName = "Id", Value = model.Id }, new SearchDocumentField { FieldName = "City", Value = model.City }, new SearchDocumentField { FieldName = "HouseNo", Value = model.HouseNo }, new SearchDocumentField { FieldName = "Name", Value = model.Name }, new SearchDocumentField { FieldName = "Street", Value = model.Street } } }; result.Add(doc); } } return(result); }
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddContentLink() { var entry = GetCatalogEntryRow("Product"); var document = new SearchDocument(); _subject.UpdateSearchDocument(ref document, entry, "en"); document["content_link"].Should().Equals(GetContentReference(444, CatalogContentType.CatalogEntry).ToString()); }
private async Task UpdateIndicesWithAddedDocuments(DateTimeOffset lastSyncPoint, DateTimeOffset currentSyncPoint, int numberOfUpdates, IndexWriter indexWriter, CancellationToken cancellationToken) { using (var dbConnection = await _SQLservice.GetConnection(cancellationToken)) { var startRow = 1; var dbCountCommand = @"SELECT Id, ISNULL(Name,''), ISNULL(Content,''), UpdatedAt from [dbo].[Test_Data] WHERE UpdatedAt >= @lastSyncPoint AND UpdatedAt < @currentSyncPoint AND DeletedAt IS NULL ORDER BY Id ASC OFFSET @StartRow - 1 ROWS FETCH NEXT @RowsPerPage ROWS ONLY "; while (numberOfUpdates >= startRow) { cancellationToken.ThrowIfCancellationRequested(); var cmd = new SqlCommand(dbCountCommand, dbConnection); cmd.Parameters.Add("@lastSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@lastSyncPoint"].Value = lastSyncPoint; cmd.Parameters.Add("@currentSyncPoint", System.Data.SqlDbType.DateTimeOffset); cmd.Parameters["@currentSyncPoint"].Value = currentSyncPoint; cmd.Parameters.Add("@StartRow", System.Data.SqlDbType.Int); cmd.Parameters["@StartRow"].Value = startRow; cmd.Parameters.Add("@RowsPerPage", System.Data.SqlDbType.Int); cmd.Parameters["@RowsPerPage"].Value = PageSize; try { using (var reader = await _SQLservice.ExecuteReaderWithRetryAsync(cmd, System.Data.CommandBehavior.SequentialAccess, cancellationToken)) { while (await reader.ReadAsync()) { var document_id = await reader.GetFieldValueAsync <int>(0); var document_name = await reader.GetFieldValueAsync <string>(1); var document_content = await reader.GetFieldValueAsync <string>(2); var document_updatedAt = await reader.GetFieldValueAsync <DateTimeOffset>(3); var updatedAtAsNumber = int.Parse(document_updatedAt.ToString("yyyyMMdd")); var searchDocument = new SearchDocument() { DocumentID = document_id.ToString(), Name = document_name, Content = document_content, UpdatedAt = updatedAtAsNumber }; var doc = new Lucene.Net.Documents.Document { // StringField indexes but doesn't tokenize new Lucene.Net.Documents.StringField("doc_id", searchDocument.DocumentID, Lucene.Net.Documents.Field.Store.YES), new Lucene.Net.Documents.StringField("name", searchDocument.Name, Lucene.Net.Documents.Field.Store.YES), new Lucene.Net.Documents.TextField("content", searchDocument.Content, Lucene.Net.Documents.Field.Store.YES), new Lucene.Net.Documents.Int32Field("updated", searchDocument.UpdatedAt, Lucene.Net.Documents.Field.Store.YES) }; indexWriter.AddDocument(doc); startRow++; } } _logger.LogDebug("Processed {0} records (of {1} total) for update", (startRow - 1), numberOfUpdates); } catch (Exception ex) { _logger.LogError(ex, "unexpected failure during indexes update"); throw; } } _logger.LogInformation("Processed {0} records for update of FTS indices. Completed.", (startRow - 1)); } }
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddDisplayName() { var entry = GetCatalogEntryRow("Product"); var document = new SearchDocument(); _subject.UpdateSearchDocument(ref document, entry, "en"); document["displayname"].Should().Equals("DisplayName"); }
public JObject IngestESData(SearchDocument document, string token) { return(_bomSearchDataAccess.IngestESData(document, token)); }
public void UpdateSearchDocument_WhenPopulatingDocument_ShouldAddTopCategory() { var entry = GetCatalogEntryRow("Product"); var document = new SearchDocument(); _subject.UpdateSearchDocument(ref document, entry, "en"); document["top_category_name"].Should().Equals("Catalog"); }
public IHttpActionResult SearchDocuments(SearchDocument objS) { TranslatorBusinessService objTBS = new TranslatorBusinessService(); return(Ok(objTBS.Documents(objS))); }