/// ----------------------------------------------------------------------------- /// <summary> /// GetSearchResults gets the search results for a passed in criteria string /// </summary> /// <remarks> /// </remarks> /// <param name="portalId">A Id of the Portal</param> /// <param name="criteria">The criteria string</param> /// ----------------------------------------------------------------------------- public override SearchResultsInfoCollection GetSearchResults(int portalId, string criteria) { bool hasExcluded = Null.NullBoolean; bool hasMandatory = Null.NullBoolean; var portal = PortalController.Instance.GetPortal(portalId); //Get the Settings for this Portal var portalSettings = new PortalSettings(portal); //We will assume that the content is in the locale of the Portal Hashtable commonWords = GetCommonWords(portalSettings.DefaultLanguage); //clean criteria criteria = criteria.ToLower(); //split search criteria into words var searchWords = new SearchCriteriaCollection(criteria); var searchResults = new Dictionary <string, SearchResultsInfoCollection>(); //dicResults is a Dictionary(Of SearchItemID, Dictionary(Of TabID, SearchResultsInfo) var dicResults = new Dictionary <int, Dictionary <int, SearchResultsInfo> >(); //iterate through search criteria words foreach (SearchCriteria criterion in searchWords) { if (commonWords.ContainsKey(criterion.Criteria) == false || portalSettings.SearchIncludeCommon) { if (!searchResults.ContainsKey(criterion.Criteria)) { searchResults.Add(criterion.Criteria, SearchDataStoreController.GetSearchResults(portalId, criterion.Criteria)); } if (searchResults.ContainsKey(criterion.Criteria)) { foreach (SearchResultsInfo result in searchResults[criterion.Criteria]) { //Add results to dicResults if (!criterion.MustExclude) { if (dicResults.ContainsKey(result.SearchItemID)) { //The Dictionary exists for this SearchItemID already so look in the TabId keyed Sub-Dictionary Dictionary <int, SearchResultsInfo> dic = dicResults[result.SearchItemID]; if (dic.ContainsKey(result.TabId)) { //The sub-Dictionary contains the item already so update the relevance SearchResultsInfo searchResult = dic[result.TabId]; searchResult.Relevance += result.Relevance; } else { //Add Entry to Sub-Dictionary dic.Add(result.TabId, result); } } else { //Create new TabId keyed Dictionary var dic = new Dictionary <int, SearchResultsInfo>(); dic.Add(result.TabId, result); //Add new Dictionary to SearchResults dicResults.Add(result.SearchItemID, dic); } } } } } } foreach (SearchCriteria criterion in searchWords) { var mandatoryResults = new Dictionary <int, bool>(); var excludedResults = new Dictionary <int, bool>(); if (searchResults.ContainsKey(criterion.Criteria)) { foreach (SearchResultsInfo result in searchResults[criterion.Criteria]) { if (criterion.MustInclude) { //Add to mandatory results lookup mandatoryResults[result.SearchItemID] = true; hasMandatory = true; } else if (criterion.MustExclude) { //Add to exclude results lookup excludedResults[result.SearchItemID] = true; hasExcluded = true; } } } foreach (KeyValuePair <int, Dictionary <int, SearchResultsInfo> > kvpResults in dicResults) { //The key of this collection is the SearchItemID, Check if the value of this collection should be processed if (hasMandatory && (!mandatoryResults.ContainsKey(kvpResults.Key))) { //1. If mandatoryResults exist then only process if in mandatoryResults Collection foreach (SearchResultsInfo result in kvpResults.Value.Values) { result.Delete = true; } } else if (hasExcluded && (excludedResults.ContainsKey(kvpResults.Key))) { //2. Do not process results in the excludedResults Collection foreach (SearchResultsInfo result in kvpResults.Value.Values) { result.Delete = true; } } } } //Process results against permissions and mandatory and excluded results var results = new SearchResultsInfoCollection(); foreach (KeyValuePair <int, Dictionary <int, SearchResultsInfo> > kvpResults in dicResults) { foreach (SearchResultsInfo result in kvpResults.Value.Values) { if (!result.Delete) { //Check If authorised to View Tab TabInfo objTab = TabController.Instance.GetTab(result.TabId, portalId, false); if (TabPermissionController.CanViewPage(objTab)) { //Check If authorised to View Module ModuleInfo objModule = ModuleController.Instance.GetModule(result.ModuleId, result.TabId, false); if (ModulePermissionController.CanViewModule(objModule)) { results.Add(result); } } } } } //Return Search Results Collection return(results); }
/// ----------------------------------------------------------------------------- /// <summary> /// AddIndexWords adds the Index Words to the Data Store /// </summary> /// <remarks> /// </remarks> /// <param name="indexId">The Id of the SearchItem</param> /// <param name="searchItem">The SearchItem</param> /// <param name="language">The Language of the current Item</param> /// <history> /// [cnurse] 11/15/2004 documented /// [cnurse] 11/16/2004 replaced calls to separate content clean-up /// functions with new call to HtmlUtils.Clean(). /// replaced logic to determine whether word should /// be indexed by call to CanIndexWord() /// [vnguyen] 09/03/2010 added searchitem title to the content and /// also tab title, description, keywords where the /// content resides for indexed searching /// </history> /// ----------------------------------------------------------------------------- private void AddIndexWords(int indexId, SearchItemInfo searchItem, string language) { //Get the Search Settings for this Portal var settings = new SearchConfig(SearchDataStoreController.GetSearchSettings(searchItem.ModuleId)); var IndexWords = new Dictionary <string, int>(); var IndexPositions = new Dictionary <string, List <int> >(); string Content = GetSearchContent(searchItem); string title = HtmlUtils.StripPunctuation(searchItem.Title, true); // Tab and Module Metadata // Retreive module and page names ModuleInfo objModule = new ModuleController().GetModule(searchItem.ModuleId); TabInfo objTab = new TabController().GetTab(objModule.TabID, objModule.PortalID, false); string tabName = HtmlUtils.StripPunctuation(objTab.TabName, true); string tabTitle = HtmlUtils.StripPunctuation(objTab.Title, true); string tabDescription = HtmlUtils.StripPunctuation(objTab.Description, true); string tabKeywords = HtmlUtils.StripPunctuation(objTab.KeyWords, true); string tagfilter = PortalController.GetPortalSetting("SearchIncludedTagInfoFilter", objModule.PortalID, Host.SearchIncludedTagInfoFilter); // clean content Content = HtmlUtils.CleanWithTagInfo(Content, tagfilter, true); // append tab and module metadata Content = Content.ToLower() + title.ToLower() + " " + tabName.ToLower() + " " + tabTitle.ToLower() + " " + tabDescription.ToLower() + " " + tabKeywords.ToLower(); // split content into words string[] ContentWords = Content.Split(' '); //process each word int intWord = 0; foreach (string strWord in ContentWords) { if (CanIndexWord(strWord, language, settings)) { var encodedWord = HttpUtility.HtmlEncode(strWord); intWord = intWord + 1; if (IndexWords.ContainsKey(encodedWord) == false) { IndexWords.Add(encodedWord, 0); IndexPositions.Add(encodedWord, new List <int>()); } //track number of occurrences of word in content IndexWords[encodedWord] = IndexWords[encodedWord] + 1; //track positions of word in content IndexPositions[encodedWord].Add(intWord); } } //get list of words ( non-common ) Hashtable Words = GetSearchWords(); //this could be cached int WordId; //iterate through each indexed word foreach (object objWord in IndexWords.Keys) { string strWord = Convert.ToString(objWord); if (Words.ContainsKey(strWord)) { //word is in the DataStore WordId = Convert.ToInt32(Words[strWord]); } else { //add the word to the DataStore WordId = DataProvider.Instance().AddSearchWord(strWord); Words.Add(strWord, WordId); } //add the indexword int SearchItemWordID = DataProvider.Instance().AddSearchItemWord(indexId, WordId, IndexWords[strWord]); string strPositions = Null.NullString; foreach (int position in IndexPositions[strWord]) { strPositions += position + ","; } DataProvider.Instance().AddSearchItemWordPosition(SearchItemWordID, strPositions); } }
/// ----------------------------------------------------------------------------- /// <summary> /// GetSearchItems gets a collection of Search Items for a Module/Tab/Portal /// </summary> /// <remarks> /// </remarks> /// <param name="portalId">A Id of the Portal</param> /// <param name="tabId">A Id of the Tab</param> /// <param name="moduleId">A Id of the Module</param> /// ----------------------------------------------------------------------------- public override SearchResultsInfoCollection GetSearchItems(int portalId, int tabId, int moduleId) { return(SearchDataStoreController.GetSearchResults(portalId, tabId, moduleId)); }
/// ----------------------------------------------------------------------------- /// <summary> /// StoreSearchItems adds the Search Item to the Data Store /// </summary> /// <remarks> /// </remarks> /// <param name="SearchItems">A Collection of SearchItems</param> /// <history> /// [cnurse] 11/15/2004 documented /// [vnguyen] 09/07/2010 Modified: Added a date comparison for LastModifiedDate on the Tab /// </history> /// ----------------------------------------------------------------------------- public override void StoreSearchItems(SearchItemInfoCollection SearchItems) { //For now as we don't support Localized content - set the locale to the default locale. This //is to avoid the error in GetDefaultLanguageByModule which artificially limits the number //of modules that can be indexed. This will need to be addressed when we support localized content. var Modules = new Dictionary <int, string>(); foreach (SearchItemInfo item in SearchItems) { if (!Modules.ContainsKey(item.ModuleId)) { Modules.Add(item.ModuleId, "en-US"); } } var objTabs = new TabController(); var objModule = new ModuleInfo(); var objTab = new TabInfo(); SearchItemInfo searchItem; Dictionary <string, SearchItemInfo> indexedItems; SearchItemInfoCollection moduleItems; //Process the SearchItems by Module to reduce Database hits foreach (KeyValuePair <int, string> kvp in Modules) { indexedItems = SearchDataStoreController.GetSearchItems(kvp.Key); //Get the Module's SearchItems to compare moduleItems = SearchItems.ModuleItems(kvp.Key); //remove deleted indexed items var moduleItemList = moduleItems.Cast <SearchItemInfo>().ToList(); indexedItems.Values .Where(i => moduleItemList.All(s => s.SearchKey != i.SearchKey)) .ToList().ForEach(i => SearchDataStoreController.DeleteSearchItem(i.SearchItemId)); //As we will be potentially removing items from the collection iterate backwards for (int iSearch = moduleItems.Count - 1; iSearch >= 0; iSearch += -1) { searchItem = moduleItems[iSearch]; //Get item from Indexed collection SearchItemInfo indexedItem = null; if (indexedItems.TryGetValue(searchItem.SearchKey, out indexedItem)) { //Get the tab where the search item resides -- used in date comparison objModule = new ModuleController().GetModule(searchItem.ModuleId); objTab = objTabs.GetTab(searchItem.TabId, objModule.PortalID, false); //Item exists so compare Dates to see if modified if (indexedItem.PubDate < searchItem.PubDate || indexedItem.PubDate < objModule.LastModifiedOnDate || indexedItem.PubDate < objTab.LastModifiedOnDate) { try { if (searchItem.PubDate < objModule.LastModifiedOnDate) { searchItem.PubDate = objModule.LastModifiedOnDate; } if (searchItem.PubDate < objTab.LastModifiedOnDate) { searchItem.PubDate = objTab.LastModifiedOnDate; } //Content modified so update SearchItem and delete item's Words Collection searchItem.SearchItemId = indexedItem.SearchItemId; SearchDataStoreController.UpdateSearchItem(searchItem); SearchDataStoreController.DeleteSearchItemWords(searchItem.SearchItemId); //re-index the content AddIndexWords(searchItem.SearchItemId, searchItem, kvp.Value); } catch (Exception ex) { //Log Exception Exceptions.Exceptions.LogException(ex); } } //Remove Items from both collections indexedItems.Remove(searchItem.SearchKey); SearchItems.Remove(searchItem); } else { try { //Item doesn't exist so Add to Index int indexID = SearchDataStoreController.AddSearchItem(searchItem); //index the content AddIndexWords(indexID, searchItem, kvp.Value); } catch (Exception ex) { //Exception is probably a duplicate key error which is probably due to bad module data Exceptions.Exceptions.LogSearchException(new SearchException(ex.Message, ex, searchItem)); } } } } }
/// ----------------------------------------------------------------------------- /// <summary> /// GetSearchItems gets a collection of Search Items for a Module/Tab/Portal /// </summary> /// <remarks> /// </remarks> /// <param name="PortalID">A Id of the Portal</param> /// <param name="TabID">A Id of the Tab</param> /// <param name="ModuleID">A Id of the Module</param> /// <history> /// [cnurse] 11/15/2004 documented /// </history> /// ----------------------------------------------------------------------------- public override SearchResultsInfoCollection GetSearchItems(int PortalID, int TabID, int ModuleID) { return(SearchDataStoreController.GetSearchResults(PortalID, TabID, ModuleID)); }
/// <summary> /// StoreSearchItems adds the Search Item to the Data Store /// </summary> /// <param name="SearchItems">A Collection of SearchItems</param> /// <history> /// [cnurse] 11/15/2004 documented /// </history> public override void StoreSearchItems(SearchItemInfoCollection SearchItems) { int i; //Get the default Search Settings _defaultSettings = Globals.HostSettings; //For now as we don't support Localized content - set the locale to the default locale. This //is to avoid the error in GetDefaultLanguageByModule which artificially limits the number //of modules that can be indexed. This will need to be addressed when we support localized content. Hashtable Modules = new Hashtable(); for (i = 0; i <= SearchItems.Count - 1; i++) { if (!Modules.ContainsKey(SearchItems[i].ModuleId.ToString())) { Modules.Add(SearchItems[i].ModuleId.ToString(), "en-US"); } } //Process the SearchItems by Module to reduce Database hits IDictionaryEnumerator moduleEnumerator = Modules.GetEnumerator(); while (moduleEnumerator.MoveNext()) { int ModuleId = Convert.ToInt32(moduleEnumerator.Key); string Language = Convert.ToString(moduleEnumerator.Value); //Get the Indexed Items that are in the Database for this Module SearchItemInfoCollection IndexedItems = GetSearchItems(ModuleId); //Get the Module's SearchItems to compare SearchItemInfoCollection ModuleItems = SearchItems.ModuleItems(ModuleId); //As we will be potentially removing items from the collection iterate backwards for (int iSearch = ModuleItems.Count - 1; iSearch >= 0; iSearch--) { SearchItemInfo SearchItem = ModuleItems[iSearch]; bool ItemFound = false; //Iterate through Indexed Items foreach (SearchItemInfo IndexedItem in IndexedItems) { //Compare the SearchKeys if (SearchItem.SearchKey == IndexedItem.SearchKey) { //Item exists so compare Dates to see if modified if (IndexedItem.PubDate < SearchItem.PubDate) { try { //Content modified so update SearchItem and delete item's Words Collection SearchItem.SearchItemId = IndexedItem.SearchItemId; SearchDataStoreController.UpdateSearchItem(SearchItem); SearchDataStoreController.DeleteSearchItemWords(SearchItem.SearchItemId); // re-index the content AddIndexWords(SearchItem.SearchItemId, SearchItem, Language); } catch (Exception ex) { //Log Exception Exceptions.Exceptions.LogException(ex); } } //Remove Items from both collections IndexedItems.Remove(IndexedItem); ModuleItems.Remove(SearchItem); //Exit the Iteration as Match found ItemFound = true; break; } } if (!ItemFound) { try { //Item doesn't exist so Add to Index int IndexID = SearchDataStoreController.AddSearchItem(SearchItem); // index the content AddIndexWords(IndexID, SearchItem, Language); } catch (Exception) { //Log Exception //LogException(ex) ** this exception has been suppressed because it fills up the event log with duplicate key errors - we still need to understand what causes it though } } } //As we removed the IndexedItems as we matched them the remaining items are deleted Items //ie they have been indexed but are no longer present Hashtable ht = new Hashtable(); foreach (SearchItemInfo IndexedItem in IndexedItems) { try { //dedupe if (ht[IndexedItem.SearchItemId] == null) { SearchDataStoreController.DeleteSearchItem(IndexedItem.SearchItemId); ht.Add(IndexedItem.SearchItemId, 0); } } catch (Exception ex) { //Log Exception Exceptions.Exceptions.LogException(ex); } } } }
/// <summary> /// AddIndexWords adds the Index Words to the Data Store /// </summary> /// <param name="indexId">The Id of the SearchItem</param> /// <param name="searchItem">The SearchItem</param> /// <param name="language">The Language of the current Item</param> /// <history> /// [cnurse] 11/15/2004 documented /// [cnurse] 11/16/2004 replaced calls to separate content clean-up /// functions with new call to HtmlUtils.Clean(). /// replaced logic to determine whether word should /// be indexed by call to CanIndexWord() /// </history> private void AddIndexWords(int indexId, SearchItemInfo searchItem, string language) { Hashtable IndexWords = new Hashtable(); Hashtable IndexPositions = new Hashtable(); //Get the Settings for this Module _settings = SearchDataStoreController.GetSearchSettings(searchItem.ModuleId); if (_settings == null) { //Try Host Settings _settings = Globals.HostSettings; } string setting = GetSetting("MaxSearchWordLength"); if (!String.IsNullOrEmpty(setting)) { maxWordLength = int.Parse(setting); } setting = GetSetting("MinSearchWordLength"); if (!String.IsNullOrEmpty(setting)) { minWordLength = int.Parse(setting); } setting = GetSetting("SearchIncludeCommon"); if (setting == "Y") { includeCommon = true; } setting = GetSetting("SearchIncludeNumeric"); if (setting == "N") { includeNumbers = false; } string Content = searchItem.Content; // clean content Content = HtmlUtils.Clean(Content, true); Content = Content.ToLower(); //' split content into words string[] ContentWords = Content.Split(' '); // process each word int intWord = 0; string strWord; foreach (string tempLoopVar_strWord in ContentWords) { strWord = tempLoopVar_strWord; if (CanIndexWord(strWord, language)) { intWord++; if (IndexWords.ContainsKey(strWord) == false) { IndexWords.Add(strWord, 0); IndexPositions.Add(strWord, 1); } // track number of occurrences of word in content IndexWords[strWord] = Convert.ToInt32(IndexWords[strWord]) + 1; // track positions of word in content IndexPositions[strWord] = Convert.ToString(IndexPositions[strWord]) + "," + intWord.ToString(); } } // get list of words ( non-common ) Hashtable Words = GetSearchWords(); // this could be cached int WordId; //' iterate through each indexed word object objWord; foreach (object tempLoopVar_objWord in IndexWords.Keys) { objWord = tempLoopVar_objWord; strWord = Convert.ToString(objWord); if (Words.ContainsKey(strWord)) { // word is in the DataStore WordId = Convert.ToInt32(Words[strWord]); } else { // add the word to the DataStore WordId = DataProvider.Instance().AddSearchWord(strWord); Words.Add(strWord, WordId); } // add the indexword int SearchItemWordID = DataProvider.Instance().AddSearchItemWord(indexId, WordId, Convert.ToInt32(IndexWords[strWord])); DataProvider.Instance().AddSearchItemWordPosition(SearchItemWordID, Convert.ToString(IndexPositions[strWord])); } }
/// <summary> /// GetSearchResults gets the search results for a passed in criteria string /// </summary> /// <param name="PortalID">A Id of the Portal</param> /// <param name="Criteria">The criteria string</param> /// <history> /// [cnurse] 11/15/2004 documented /// </history> public override SearchResultsInfoCollection GetSearchResults(int PortalID, string Criteria) { //We will assume that the content is in the locale of the Portal PortalController objPortalController = new PortalController(); PortalInfo objPortal = objPortalController.GetPortal(PortalID); string locale = objPortal.DefaultLanguage; Hashtable CommonWords = GetCommonWords(locale); string setting = null; //Get the default Search Settings _defaultSettings = Globals.HostSettings; //Get the Settings for this Portal ModuleController objModuleController = new ModuleController(); ModuleInfo objModule = objModuleController.GetModuleByDefinition(-1, "Search Admin"); if (objModule != null) { _settings = PortalSettings.GetModuleSettings(objModule.ModuleID); } setting = GetSetting("SearchIncludeCommon"); if (setting == "Y") { includeCommon = true; } // clean criteria Criteria = Criteria.ToLower(); // split search criteria into words SearchCriteriaCollection SearchWords = new SearchCriteriaCollection(Criteria); Hashtable SearchResults = new Hashtable(); // iterate through search criteria words SearchCriteria Criterion = null; foreach (SearchCriteria CriterionWithinLoop in SearchWords) { Criterion = CriterionWithinLoop; if (CommonWords.ContainsKey(CriterionWithinLoop.Criteria) == false || includeCommon) { SearchResultsInfoCollection ResultsCollection = SearchDataStoreController.GetSearchResults(PortalID, Criterion.Criteria); if (CriterionWithinLoop.MustExclude == false) { // Add all these to the results foreach (SearchResultsInfo Result in ResultsCollection) { if (SearchResults.ContainsKey(Result.SearchItemID)) { ((SearchResultsInfo)(SearchResults[Result.SearchItemID])).Relevance += Result.Relevance; } else { SearchResults.Add(Result.SearchItemID, Result); } } } } } // Validate MustInclude and MustExclude foreach (SearchCriteria CriterionWithinLoop in SearchWords) { Criterion = CriterionWithinLoop; SearchResultsInfoCollection ResultsCollection = SearchDataStoreController.GetSearchResults(PortalID, Criterion.Criteria); if (CriterionWithinLoop.MustInclude) { // We need to remove items which do not include this term Hashtable MandatoryResults = new Hashtable(); foreach (SearchResultsInfo Result in ResultsCollection) { MandatoryResults.Add(Result.SearchItemID, 0); } foreach (SearchResultsInfo Result in SearchResults.Values) { if (MandatoryResults.ContainsKey(Result.SearchItemID) == false) { Result.Delete = true; } } } if (CriterionWithinLoop.MustExclude) { // We need to remove items which do include this term Hashtable ExcludedResults = new Hashtable(); foreach (SearchResultsInfo Result in ResultsCollection) { ExcludedResults.Add(Result.SearchItemID, 0); } foreach (SearchResultsInfo Result in SearchResults.Values) { if (ExcludedResults.ContainsKey(Result.SearchItemID) == true) { Result.Delete = true; } } } } //Only include results we have permission to see SearchResultsInfoCollection Results = new SearchResultsInfoCollection(); TabController objTabController = new TabController(); Hashtable hashTabsAllowed = new Hashtable(); foreach (SearchResultsInfo SearchResult in SearchResults.Values) { if (!SearchResult.Delete) { //Check If authorised to View Tab Hashtable hashModulesAllowed = null; object tabAllowed = hashTabsAllowed[SearchResult.TabId]; if (tabAllowed == null) { TabInfo objTab = objTabController.GetTab(SearchResult.TabId, PortalID, false); if (PortalSecurity.IsInRoles(objTab.AuthorizedRoles)) { hashModulesAllowed = new Hashtable(); tabAllowed = hashModulesAllowed; } else { tabAllowed = 0; hashModulesAllowed = null; } hashTabsAllowed.Add(SearchResult.TabId, tabAllowed); } else { if (tabAllowed is Hashtable) { hashModulesAllowed = (Hashtable)tabAllowed; } else { hashModulesAllowed = null; } } if (hashModulesAllowed != null) { bool addResult = false; if (!(hashModulesAllowed.ContainsKey(SearchResult.ModuleId))) { //Now check if authorized to view module objModule = objModuleController.GetModule(SearchResult.ModuleId, SearchResult.TabId, false); addResult = (objModule.IsDeleted == false && PortalSecurity.IsInRoles(objModule.AuthorizedViewRoles)); hashModulesAllowed.Add(SearchResult.ModuleId, addResult); } else { addResult = Convert.ToBoolean(hashModulesAllowed[SearchResult.ModuleId]); } if (addResult) { Results.Add(SearchResult); } } } } //Return Search Results Collection return(Results); }