/// <summary> /// Get step doc index. /// </summary> /// <param name="position">first byte of the index (include step doc index)</param> /// <param name="length">index buffer length</param> /// <param name="count">count of the document records in this index</param> /// <param name="indexPostion">first byte of the index (exclude step doc index)<</param> /// <returns>step doc index. If no step doc index, return null</returns> public List <DocumentPosition> GetStepDocIndex(long position, long length, out int count, out long indexPostion) { Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport(); List <DocumentPosition> result; //indexPostion = position; _IndexFile.Seek(position, System.IO.SeekOrigin.Begin); result = DocumentPositionList.DeserializeSkipDocIndex(_IndexFile, false, out count); if (result != null) { count = DocumentPositionList.GetDocumentsCount(_IndexFile); } indexPostion = _IndexFile.Position; //The index position exclude count; performanceReport.Stop(string.Format("Read index file: len={0}, {1} results. ", _IndexFile.Position - position, count)); return(result); }
public Core.SFQL.Parse.DocumentResultWhereDictionary Search() { Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Search of Match1"); Core.SFQL.Parse.DocumentResultWhereDictionary result = new Core.SFQL.Parse.DocumentResultWhereDictionary(); if (_QueryWords.Count <= 0 || _WordIndexes.Length <= 0) { if (_QueryParameter.Not && UpDict != null) { return(UpDict); } else { return(result); } } if (this._QueryParameter.Not) { if (_InvertedIndex.IndexMode == Field.IndexMode.Simple) { Calculate(null, ref result, _WordIndexes); } else { CalculateWithPosition(null, ref result, _WordIndexes); } } else { if (_InvertedIndex.IndexMode == Field.IndexMode.Simple) { Calculate(this.UpDict, ref result, _WordIndexes); } else { CalculateWithPosition(this.UpDict, ref result, _WordIndexes); } } if (this._QueryParameter.Not) { result.Not = true; if (UpDict != null) { result = result.AndMergeForNot(result, UpDict); } } performanceReport.Stop(); return(result); }
public Core.SFQL.Parse.DocumentResultWhereDictionary Search() { Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Search of Contains"); List <WordIndexForQuery[]> partList = GetAllPartOfWordIndexes(); if (_QueryWords.Count <= 0 || partList.Count <= 0) { return(PartSearch(new WordIndexForQuery[0])); } Core.SFQL.Parse.DocumentResultWhereDictionary result = PartSearch(partList[0]); for (int i = 1; i < partList.Count; i++) { bool someWordNoResult = false; foreach (WordIndexForQuery w in partList[i]) { if (w.WordIndex.WordCount == 0) { someWordNoResult = true; break; } w.WordIndex.Reset(); } if (!someWordNoResult) { result.OrMerge(result, PartSearch(partList[i])); } } if (this._QueryParameter.Not) { result.Not = true; if (UpDict != null) { result = result.AndMergeForNot(result, UpDict); } } performanceReport.Stop(); return(result); }
/// <summary> /// Get the specify document position list from .idx file /// </summary> /// <param name="position">the position of the first byte of the index</param> /// <param name="length">index content length</param> /// <param name="count">max count of the documents that want read</param> /// <param name="simple">is it simple index mode</param> /// <returns>Word position list</returns> public WordDocumentsList GetDocList(long position, long length, int count, bool simple) { Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport(); _IndexFile.Seek(position, System.IO.SeekOrigin.Begin); WordDocumentsList result = new WordDocumentsList(); if (count >= 0) { result.AddRange(Entity.DocumentPositionList.Deserialize(_IndexFile, ref count, simple, out result.WordCountSum)); result.RelDocCount = count; } else { result.AddRange(Entity.DocumentPositionList.Deserialize(_IndexFile, simple, out result.WordCountSum)); result.RelDocCount = result.Count; } performanceReport.Stop(string.Format("Read index file: len={0}, {1} results. ", _IndexFile.Position - position, result.Count)); return(result); }
unsafe private void Calculate(DocumentResultWhereDictionary upDict, ref DocumentResultWhereDictionary docIdRank, WordIndexForQuery[] wordIndexes) { Array.Sort(wordIndexes); MinResultCount = _DBProvider.Table.GroupByLimit; //Get max word doc list count int minWordDocListCount = 1 * 1024 * 1024; //1M foreach (WordIndexForQuery wifq in wordIndexes) { minWordDocListCount = Math.Min(minWordDocListCount, wifq.WordIndex.WordDocList.Count); } if (docIdRank.Count == 0) { if (minWordDocListCount > DocumentResultWhereDictionary.DefaultSize) { docIdRank = new Core.SFQL.Parse.DocumentResultWhereDictionary(minWordDocListCount); } } Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Calculate"); //Merge bool oneWordOptimize = this._QueryParameter.CanLoadPartOfDocs && this._QueryParameter.NoAndExpression && wordIndexes.Length == 1; int oneWordMaxCount = 0; if (oneWordOptimize) { //One word WordIndexForQuery wifq = wordIndexes[0]; //first word //Entity.DocumentPositionList[] wifqDocBuf = wifq.WordIndex.DocPositionBuf; Entity.DocumentPositionList docList = wifq.WordIndex.GetNext(); int j = 0; while (docList.DocumentId >= 0) { //Entity.DocumentPositionList docList = wifq.WordIndex[j]; Core.SFQL.Parse.DocumentResultPoint drp; drp.pDocumentResult = null; if (j > MinResultCount) { if (oneWordMaxCount > docList.Count) { j++; docList = wifq.WordIndex.GetNext(); continue; } } else { if (oneWordMaxCount < docList.Count) { oneWordMaxCount = docList.Count; } } long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } if (upDict == null) { docIdRank.Add(docList.DocumentId, score); } else { if (!upDict.Not) { if (upDict.ContainsKey(docList.DocumentId)) { docIdRank.Add(docList.DocumentId, score); } } else { if (!upDict.ContainsKey(docList.DocumentId)) { docIdRank.Add(docList.DocumentId, score); } } } j++; docList = wifq.WordIndex.GetNext(); } } else { int wordIndexesLen = wordIndexes.Length; WordIndexForQuery fstWifq = wordIndexes[0]; //first word Entity.DocumentPositionList fstDocList = fstWifq.WordIndex.GetNext(); Entity.DocumentPositionList[] docListArr = new Hubble.Core.Entity.DocumentPositionList[wordIndexesLen]; docListArr[0] = fstDocList; while (fstDocList.DocumentId >= 0) { int curWord = 1; int firstDocId = fstDocList.DocumentId; while (curWord < wordIndexesLen) { docListArr[curWord] = wordIndexes[curWord].WordIndex.Get(firstDocId); if (docListArr[curWord].DocumentId < 0) { break; } curWord++; } //While if (curWord >= wordIndexesLen) { //Matched long totalScore = 0; for (int i = 0; i < wordIndexesLen; i++) { WordIndexForQuery wifq = wordIndexes[i]; Entity.DocumentPositionList docList = docListArr[i]; long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } totalScore += score; } if (upDict == null) { docIdRank.Add(firstDocId, totalScore); } else { if (!upDict.Not) { if (upDict.ContainsKey(firstDocId)) { docIdRank.Add(firstDocId, totalScore); } } else { if (!upDict.ContainsKey(firstDocId)) { docIdRank.Add(firstDocId, totalScore); } } } } fstDocList = fstWifq.WordIndex.GetNext(); docListArr[0] = fstDocList; } } //Merge score if upDict != null if (upDict != null) { if (!upDict.Not) { foreach (int docid in docIdRank.Keys) { DocumentResult *upDrp; if (upDict.TryGetValue(docid, out upDrp)) { DocumentResult *drpResult; if (docIdRank.TryGetValue(docid, out drpResult)) { drpResult->Score += upDrp->Score; } } } } } DeleteProvider delProvider = _DBProvider.DelProvider; int delCount = delProvider.Filter(docIdRank); if (oneWordOptimize && _QueryParameter.CanLoadPartOfDocs && upDict == null) { docIdRank.RelTotalCount = wordIndexes[0].RelTotalCount - delCount; } else { docIdRank.RelTotalCount = docIdRank.Count; } performanceReport.Stop(); }
public Core.SFQL.Parse.DocumentResultWhereDictionary Search() { Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Search of Like"); Core.SFQL.Parse.DocumentResultWhereDictionary result; if (_QueryThroughDB) { string orderBy = _QueryParameter.OrderBy; if (orderBy != null) { if (orderBy.ToLower().Contains("score")) { orderBy = null; } else if (_DBProvider.Table.IndexOnly && _DBProvider.Table.DocIdReplaceField != null && orderBy.ToLower().Contains("docid")) { orderBy = null; } } if (!_QueryParameter.NoAndExpression || UpDict != null) { if (_DBProvider.Table.IndexOnly && _DBProvider.Table.DocIdReplaceField != null) { orderBy = _DBProvider.Table.DocIdReplaceField; } else { orderBy = "docid"; } _QueryParameter.End = -1; } result = _DBProvider.DBAdapter.GetDocumentResults(_QueryParameter.End, string.Format("{0} like '{1}'", _FieldName, _LikeString), orderBy); if (UpDict != null) { if (!UpDict.Not) { result = result.AndMergeDict(result, UpDict); } } } else { List <WordIndexForQuery[]> partList = GetAllPartOfWordIndexes(); if (_QueryWords.Count <= 0 || partList.Count <= 0) { return(PartSearch(new WordIndexForQuery[0])); } result = PartSearch(partList[0]); for (int i = 1; i < partList.Count; i++) { bool someWordNoResult = false; foreach (WordIndexForQuery w in partList[i]) { if (w.WordIndex.WordCount == 0) { someWordNoResult = true; break; } w.WordIndex.Reset(); } if (!someWordNoResult) { result.OrMerge(result, PartSearch(partList[i])); } } } if (this._QueryParameter.Not) { result.Not = true; if (UpDict != null) { result = result.AndMergeForNot(result, UpDict); } } if (!_QueryThroughDB) { result = GetExternLikeDocResults(result); } performanceReport.Stop(); return(result); }
unsafe private void CalculateWithPosition(Core.SFQL.Parse.DocumentResultWhereDictionary upDict, ref Core.SFQL.Parse.DocumentResultWhereDictionary docIdRank, WordIndexForQuery[] wordIndexes) { if (UseMatch(wordIndexes)) { CalculateWithPositionMatch(upDict, ref docIdRank, wordIndexes); return; } Array.Sort(wordIndexes); AdjustSort(wordIndexes); MinResultCount = _DBProvider.Table.GroupByLimit; //Get max word doc list count int minWordDocListCount = 1 * 1024 * 1024; //1M foreach (WordIndexForQuery wifq in wordIndexes) { minWordDocListCount = Math.Min(minWordDocListCount, wifq.WordIndex.Count); } Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Calculate"); //Merge bool oneWordOptimize = this._QueryParameter.CanLoadPartOfDocs && this._QueryParameter.AndExpressionCanBeOptimized(_DBProvider) && wordIndexes.Length == 1 && _NotInDict == null && _QueryParameter.End >= 0 && !_QueryParameter.NeedDistinct; if (oneWordOptimize) { IQueryOptimize qOptimize = QueryOptimizeBuilder.Build(typeof(OneWordOptimize), DBProvider, _QueryParameter.End, _QueryParameter.OrderBy, _QueryParameter.OrderBys, _QueryParameter.NeedGroupBy, _QueryParameter.OrderByCanBeOptimized, _QueryParameter.NeedFilterUntokenizedConditions(this._DBProvider), _QueryParameter.UntokenizedTreeOnRoot, wordIndexes); try { qOptimize.CalculateOptimize(upDict, ref docIdRank); return; } finally { performanceReport.Stop(); } } if (this._QueryParameter.CanLoadPartOfDocs && this._QueryParameter.AndExpressionCanBeOptimized(_DBProvider) && _NotInDict == null && _QueryParameter.End >= 0 && !_QueryParameter.NeedDistinct) { IQueryOptimize qOptimize = QueryOptimizeBuilder.Build(typeof(ContainsOptimize), DBProvider, _QueryParameter.End, _QueryParameter.OrderBy, _QueryParameter.OrderBys, _QueryParameter.NeedGroupBy, _QueryParameter.OrderByCanBeOptimized, _QueryParameter.NeedFilterUntokenizedConditions(this._DBProvider), _QueryParameter.UntokenizedTreeOnRoot, wordIndexes); try { qOptimize.CalculateOptimize(upDict, ref docIdRank); return; } finally { performanceReport.Stop(); } //if (qOptimize.Argument.IsOrderByScoreDesc()) //{ //} } if (docIdRank.Count == 0) { if (minWordDocListCount > DocumentResultWhereDictionary.DefaultSize) { docIdRank = new Core.SFQL.Parse.DocumentResultWhereDictionary(minWordDocListCount); } } { double ratio = 1; if (wordIndexes.Length > 1) { ratio = (double)2 / (double)(wordIndexes.Length - 1); } int wordIndexesLen = wordIndexes.Length; WordIndexForQuery fstWifq = wordIndexes[0]; //first word OriginalDocumentPositionList fstODPL = new OriginalDocumentPositionList(); fstWifq.WordIndex.GetNextOriginal(ref fstODPL); //Entity.DocumentPositionList fstDocList = fstWifq.WordIndex.GetNext(); Entity.DocumentPositionList[] docListArr = new Hubble.Core.Entity.DocumentPositionList[wordIndexesLen]; //docListArr[0] = fstDocList; fstODPL.ToDocumentPositionList(ref docListArr[0]); OriginalDocumentPositionList odpl = new OriginalDocumentPositionList(); while (fstODPL.DocumentId >= 0) { int curWord = 1; int firstDocId = fstODPL.DocumentId; while (curWord < wordIndexesLen) { //docListArr[curWord] = wordIndexes[curWord].WordIndex.Get(firstDocId); wordIndexes[curWord].WordIndex.GetNextOriginalWithDocId(ref odpl, firstDocId); odpl.ToDocumentPositionList(ref docListArr[curWord]); if (docListArr[curWord].DocumentId < 0) { if ((wordIndexes[curWord].Flags & WordInfo.Flag.Or) != 0) { curWord++; continue; } else { break; } } curWord++; } //While if (curWord >= wordIndexesLen) { //Matched //Caculate score long totalScore = 0; Entity.DocumentPositionList lastDocList = new Hubble.Core.Entity.DocumentPositionList(); for (int i = 0; i < wordIndexesLen; i++) { WordIndexForQuery wifq = wordIndexes[i]; if (wifq.WordIndex.Count == 0) { //a^5000^0 b^5000^2^1 //if has a and hasn't b but b can be or //2010-09-30 eaglet continue; } Entity.DocumentPositionList docList = docListArr[i]; long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } double delta = 1; if (i > 0) { //Calculate with position double queryPositionDelta = wifq.FirstPosition - wordIndexes[i - 1].FirstPosition; double positionDelta = docList.FirstPosition - lastDocList.FirstPosition; delta = Math.Abs(queryPositionDelta - positionDelta); if (delta < 0.031) { delta = 0.031; } else if (delta <= 1.1) { delta = 0.5; } else if (delta <= 2.1) { delta = 1; } delta = Math.Pow((1 / delta), ratio) * docList.Count * lastDocList.Count / (double)(wifq.QueryCount * wordIndexes[i - 1].QueryCount); } lastDocList = docList; totalScore += (long)(score * delta); } bool notInDict = false; if (_NotInDict != null) { if (_NotInDict.ContainsKey(firstDocId)) { notInDict = true; } } if (!notInDict) { if (upDict == null) { docIdRank.Add(firstDocId, totalScore); } else { if (!upDict.Not) { if (upDict.ContainsKey(firstDocId)) { docIdRank.Add(firstDocId, totalScore); } } else { if (!upDict.ContainsKey(firstDocId)) { docIdRank.Add(firstDocId, totalScore); } } } } }//if (curWord >= wordIndexesLen) //fstDocList = fstWifq.WordIndex.GetNext(); //docListArr[0] = fstDocList; fstWifq.WordIndex.GetNextOriginal(ref fstODPL); fstODPL.ToDocumentPositionList(ref docListArr[0]); } } //Merge score if upDict != null if (upDict != null) { if (!upDict.Not) { foreach (int docid in docIdRank.Keys) { DocumentResult *upDrp; if (upDict.TryGetValue(docid, out upDrp)) { DocumentResult *drpResult; if (docIdRank.TryGetValue(docid, out drpResult)) { drpResult->Score += upDrp->Score; } } } } } DeleteProvider delProvider = _DBProvider.DelProvider; int delCount = delProvider.Filter(docIdRank); if (oneWordOptimize && _QueryParameter.CanLoadPartOfDocs && upDict == null) { docIdRank.RelTotalCount = wordIndexes[0].RelTotalCount - delCount; } else { docIdRank.RelTotalCount = docIdRank.Count; } performanceReport.Stop(); }
unsafe private void CalculateWithPositionMatch(Core.SFQL.Parse.DocumentResultWhereDictionary upDict, ref Core.SFQL.Parse.DocumentResultWhereDictionary docIdRank, WordIndexForQuery[] wordIndexes) { Array.Sort(wordIndexes); MinResultCount = _DBProvider.Table.GroupByLimit; double ratio = 1; if (wordIndexes.Length > 1) { ratio = (double)2 / (double)(wordIndexes.Length - 1); } //Get max word doc list count int maxWordDocListCount = 0; int documentSum = 0; foreach (WordIndexForQuery wifq in wordIndexes) { maxWordDocListCount += wifq.WordIndex.Count; } maxWordDocListCount += maxWordDocListCount / 2; if (maxWordDocListCount > 1024 * 1024) { maxWordDocListCount = 1024 * 1024; } if (docIdRank.Count == 0) { if (maxWordDocListCount > DocumentResultWhereDictionary.DefaultSize) { docIdRank = new Core.SFQL.Parse.DocumentResultWhereDictionary(maxWordDocListCount); } } Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Calculate"); //Merge bool oneWordOptimize = this._QueryParameter.CanLoadPartOfDocs && this._QueryParameter.NoAndExpression && wordIndexes.Length == 1; for (int i = 0; i < wordIndexes.Length; i++) { WordIndexForQuery wifq = wordIndexes[i]; //Entity.DocumentPositionList[] wifqDocBuf = wifq.WordIndex.DocPositionBuf; Entity.DocumentPositionList docList = wifq.WordIndex.GetNext(); int j = 0; int oneWordMaxCount = 0; while (docList.DocumentId >= 0) { Core.SFQL.Parse.DocumentResultPoint drp; drp.pDocumentResult = null; if (oneWordOptimize) { if (j > MinResultCount) { if (oneWordMaxCount > docList.Count) { docList = wifq.WordIndex.GetNext(); j++; continue; } } else { if (oneWordMaxCount < docList.Count) { oneWordMaxCount = docList.Count; } } } if (j > wifq.RelTotalCount) { break; } long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } bool exits = drp.pDocumentResult != null; if (!exits && i > 0) { exits = docIdRank.TryGetValue(docList.DocumentId, out drp); } if (exits) { drp.pDocumentResult->Score += score; double queryPositionDelta = wifq.FirstPosition - drp.pDocumentResult->LastWordIndexFirstPosition; double positionDelta = docList.FirstPosition - drp.pDocumentResult->LastPosition; double delta = Math.Abs(queryPositionDelta - positionDelta); if (delta < 0.031) { delta = 0.031; } else if (delta <= 1.1) { delta = 0.5; } else if (delta <= 2.1) { delta = 1; } delta = Math.Pow((1 / delta), ratio) * docList.Count * drp.pDocumentResult->LastCount / (double)(wifq.QueryCount * drp.pDocumentResult->LastWordIndexQueryCount); //some words missed //if (i - drp.pDocumentResult->LastIndex > 1) //{ // int sumWordRank = 10; // for (int k = drp.pDocumentResult->LastIndex + 1; k < i; k++) // { // sumWordRank += wordIndexes[k].WordRank; // } // delta /= (double)sumWordRank; //} drp.pDocumentResult->Score = (long)(drp.pDocumentResult->Score * delta); drp.pDocumentResult->LastIndex = (UInt16)i; drp.pDocumentResult->LastPosition = docList.FirstPosition; drp.pDocumentResult->LastCount = (UInt16)docList.Count; drp.pDocumentResult->LastWordIndexFirstPosition = (UInt16)wifq.FirstPosition; } else { //some words missed //if (i > 0) //{ // int sumWordRank = 10; // for (int k = 0; k < i; k++) // { // sumWordRank += wordIndexes[k].WordRank; // } // double delta = 1 / (double)sumWordRank; // score = (long)(score * delta); //} bool notInDict = false; if (_NotInDict != null) { if (_NotInDict.ContainsKey(docList.DocumentId)) { notInDict = true; } } if (!notInDict) { if (upDict == null) { DocumentResult docResult = new DocumentResult(docList.DocumentId, score, wifq.FirstPosition, wifq.QueryCount, docList.FirstPosition, docList.Count, i); docIdRank.Add(docList.DocumentId, docResult); } else { if (!upDict.Not) { if (upDict.ContainsKey(docList.DocumentId)) { DocumentResult docResult = new DocumentResult(docList.DocumentId, score, wifq.FirstPosition, wifq.QueryCount, docList.FirstPosition, docList.Count, i); docIdRank.Add(docList.DocumentId, docResult); } } else { if (!upDict.ContainsKey(docList.DocumentId)) { DocumentResult docResult = new DocumentResult(docList.DocumentId, score, wifq.FirstPosition, wifq.QueryCount, docList.FirstPosition, docList.Count, i); docIdRank.Add(docList.DocumentId, docResult); } } } } } docList = wifq.WordIndex.GetNext(); j++; if (j > wifq.WordIndex.Count) { break; } } } //Merge score if upDict != null if (upDict != null) { if (!upDict.Not) { foreach (int docid in docIdRank.Keys) { DocumentResult *upDrp; if (upDict.TryGetValue(docid, out upDrp)) { DocumentResult *drpResult; if (docIdRank.TryGetValue(docid, out drpResult)) { drpResult->Score += upDrp->Score; } } } } } //some words missed //if (wordIndexes.Length > 1) //{ // List<DocumentResult> reduceDocs = new List<DocumentResult>(docIdRank.Count); // int lstIndex = wordIndexes.Length - 1; // foreach (Core.SFQL.Parse.DocumentResultPoint drp in docIdRank.Values) // { // DocumentResult* dr = drp.pDocumentResult; // //DocumentResult* dr1 = drp.pDocumentResult; // if (dr->LastIndex != lstIndex) // { // int sumWordRank = 10; // for (int k = dr->LastIndex + 1; k <= lstIndex; k++) // { // sumWordRank += wordIndexes[k].WordRank; // } // double delta = 1 / (double)sumWordRank; // dr->Score = (long)((double)dr->Score * delta); // } // if (dr->Score < 0) // { // dr->Score = long.MaxValue / 10; // } // } //} performanceReport.Stop(); documentSum += docIdRank.Count; if (documentSum > _TotalDocuments) { documentSum = _TotalDocuments; } DeleteProvider delProvider = _DBProvider.DelProvider; int deleteCount = delProvider.Filter(docIdRank); if (_QueryParameter.CanLoadPartOfDocs && upDict == null) { if (docIdRank.Count < wordIndexes[wordIndexes.Length - 1].RelTotalCount) { if (wordIndexes.Length > 1) { if (wordIndexes[wordIndexes.Length - 1].RelTotalCount > _DBProvider.MaxReturnCount) { documentSum += wordIndexes[wordIndexes.Length - 1].RelTotalCount - _DBProvider.MaxReturnCount; } if (documentSum > _TotalDocuments) { documentSum = _TotalDocuments; } docIdRank.RelTotalCount = documentSum; } else { docIdRank.RelTotalCount = wordIndexes[wordIndexes.Length - 1].RelTotalCount; } } } docIdRank.RelTotalCount -= deleteCount; }
unsafe private void Calculate(DocumentResultWhereDictionary upDict, ref DocumentResultWhereDictionary docIdRank, WordIndexForQuery[] wordIndexes) { Array.Sort(wordIndexes); MinResultCount = _DBProvider.Table.GroupByLimit; //Get max word doc list count int maxWordDocListCount = 0; int documentSum = 0; foreach (WordIndexForQuery wifq in wordIndexes) { maxWordDocListCount += wifq.WordIndex.RelDocCount; } if (docIdRank.Count == 0) { if (maxWordDocListCount > DocumentResultWhereDictionary.DefaultSize) { docIdRank = new Core.SFQL.Parse.DocumentResultWhereDictionary(maxWordDocListCount); } } Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Calculate"); //Merge bool oneWordOptimize = this._QueryParameter.CanLoadPartOfDocs && this._QueryParameter.NoAndExpression && wordIndexes.Length == 1; for (int i = 0; i < wordIndexes.Length; i++) { WordIndexForQuery wifq = wordIndexes[i]; //Entity.DocumentPositionList[] wifqDocBuf = wifq.WordIndex.DocPositionBuf; Entity.DocumentPositionList docList = wifq.WordIndex.GetNext(); int j = 0; int oneWordMaxCount = 0; while (docList.DocumentId >= 0) { //Entity.DocumentPositionList docList = wifq.WordIndex[j]; Core.SFQL.Parse.DocumentResultPoint drp; drp.pDocumentResult = null; if (oneWordOptimize) { if (j > MinResultCount) { if (j > MinResultCount) { if (oneWordMaxCount > docList.Count) { docList = wifq.WordIndex.GetNext(); j++; continue; } } else { if (oneWordMaxCount < docList.Count) { oneWordMaxCount = docList.Count; } } } } long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } bool exits = drp.pDocumentResult != null; if (!exits && i > 0) { exits = docIdRank.TryGetValue(docList.DocumentId, out drp); } if (exits) { drp.pDocumentResult->Score += score; } else { bool notInDict = false; if (_NotInDict != null) { if (_NotInDict.ContainsKey(docList.DocumentId)) { notInDict = true; } } if (!notInDict) { if (upDict == null) { docIdRank.Add(docList.DocumentId, score); } else { if (!upDict.Not) { if (upDict.ContainsKey(docList.DocumentId)) { docIdRank.Add(docList.DocumentId, score); } } else { if (!upDict.ContainsKey(docList.DocumentId)) { docIdRank.Add(docList.DocumentId, score); } } } } } docList = wifq.WordIndex.GetNext(); j++; } } //Merge score if upDict != null if (upDict != null) { if (!upDict.Not) { foreach (int docid in docIdRank.Keys) { DocumentResult *upDrp; if (upDict.TryGetValue(docid, out upDrp)) { DocumentResult *drpResult; if (docIdRank.TryGetValue(docid, out drpResult)) { drpResult->Score += upDrp->Score; } } } } } documentSum += docIdRank.Count; if (documentSum > _TotalDocuments) { documentSum = _TotalDocuments; } DeleteProvider delProvider = _DBProvider.DelProvider; int deleteCount = delProvider.Filter(docIdRank); if (_QueryParameter.CanLoadPartOfDocs && upDict == null) { if (docIdRank.Count < wordIndexes[wordIndexes.Length - 1].RelTotalCount) { if (wordIndexes.Length > 1) { if (wordIndexes[wordIndexes.Length - 1].RelTotalCount > _DBProvider.MaxReturnCount) { documentSum += wordIndexes[wordIndexes.Length - 1].RelTotalCount - _DBProvider.MaxReturnCount; } if (documentSum > _TotalDocuments) { documentSum = _TotalDocuments; } docIdRank.RelTotalCount = documentSum; } else { docIdRank.RelTotalCount = wordIndexes[wordIndexes.Length - 1].RelTotalCount; } } } docIdRank.RelTotalCount -= deleteCount; performanceReport.Stop(); }
/// <summary> /// Order by score desc /// and only one expression in the banch of expression tree. /// and more than two words /// </summary> /// <param name="upDict"></param> /// <param name="docIdRank"></param> /// <param name="wordIndexes"></param> unsafe private void CalculateWithPositionOrderByScoreDesc11(Core.SFQL.Parse.DocumentResultWhereDictionary upDict, ref Core.SFQL.Parse.DocumentResultWhereDictionary docIdRank, WordIndexForQuery[] wordIndexes) { if (upDict != null) { throw new ParseException("UpDict is not null!"); } Array.Sort(wordIndexes); //Calculate top int top; if (this._QueryParameter.End >= 0) { top = (1 + this._QueryParameter.End / 100) * 100; if (top <= 0) { top = 100; } //if (this._QueryParameter.End * 2 > top) //{ // top *= 2; //} } else { top = int.MaxValue; } double ratio = 1; if (wordIndexes.Length > 1) { ratio = (double)2 / (double)(wordIndexes.Length - 1); } //Get max word doc list count int maxWordDocListCount = 0; int documentSum = 0; foreach (WordIndexForQuery wifq in wordIndexes) { maxWordDocListCount += wifq.WordIndex.Count; } maxWordDocListCount += maxWordDocListCount / 2; if (maxWordDocListCount > 1024 * 1024) { maxWordDocListCount = 1024 * 1024; } Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Calculate"); bool groupbyScanAll = false; //Match for group by if (this._QueryParameter.NeedGroupBy) { groupbyScanAll = true; int groupbyContainsCount = 0; int groupbyLimit = _DBProvider.Table.GroupByLimit; BitSet bitSet = new BitSet(); for (int i = 0; i < wordIndexes.Length; i++) { WordIndexForQuery wifq = wordIndexes[i]; Entity.DocumentPositionList docList = wifq.WordIndex.GetNext(); while (docList.DocumentId >= 0) { if (bitSet.ForceAdd(docList.DocumentId)) { groupbyContainsCount++; } if (groupbyContainsCount >= groupbyLimit) { groupbyScanAll = false; break; } docList = wifq.WordIndex.GetNext(); } wifq.WordIndex.Reset(); if (!groupbyScanAll) { break; } } AscIntList groupByCollect = new AscIntList(); groupByCollect.AddRange(bitSet); } //Merge int indexInTop = 0; for (int i = 0; i < wordIndexes.Length; i++) { if (docIdRank.Count >= top) { break; } indexInTop = i; WordIndexForQuery wifq = wordIndexes[i]; //Entity.DocumentPositionList[] wifqDocBuf = wifq.WordIndex.DocPositionBuf; Entity.DocumentPositionList docList = wifq.WordIndex.GetNext(); int j = 0; while (docList.DocumentId >= 0) { Core.SFQL.Parse.DocumentResultPoint drp; drp.pDocumentResult = null; long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } else { switch (i) { case 0: score *= 20; break; case 1: score *= 4; break; case 2: score *= 1; break; case 3: score /= 2; break; default: score /= i; break; } } if (score < 0) { //Overflow score = long.MaxValue - 4000000; } bool exits = drp.pDocumentResult != null; if (!exits && i > 0) { exits = docIdRank.TryGetValue(docList.DocumentId, out drp); } if (exits) { drp.pDocumentResult->Score += score; drp.pDocumentResult->HitCount++; double queryPositionDelta = wifq.FirstPosition - drp.pDocumentResult->LastWordIndexFirstPosition; double positionDelta = docList.FirstPosition - drp.pDocumentResult->LastPosition; double delta = Math.Abs(queryPositionDelta - positionDelta); if (delta < 0.031) { delta = 0.031; } else if (delta <= 1.1) { delta = 0.5; } else if (delta <= 2.1) { delta = 1; } delta = Math.Pow((1 / delta), ratio) * docList.Count * drp.pDocumentResult->LastCount / (double)(wifq.QueryCount * drp.pDocumentResult->LastWordIndexQueryCount); drp.pDocumentResult->Score = (long)(drp.pDocumentResult->Score * delta); //Overflow, if match too much, sometime score would less than zero. if (drp.pDocumentResult->Score < 0) { drp.pDocumentResult->Score = long.MaxValue - 4000000; } drp.pDocumentResult->LastIndex = (UInt16)i; drp.pDocumentResult->LastPosition = docList.FirstPosition; drp.pDocumentResult->LastCount = (UInt16)docList.Count; drp.pDocumentResult->LastWordIndexFirstPosition = (UInt16)wifq.FirstPosition; } else { bool notInDict = false; if (_NotInDict != null) { if (_NotInDict.ContainsKey(docList.DocumentId)) { notInDict = true; } } if (!notInDict) { //upDict is null in this function DocumentResult docResult = new DocumentResult(docList.DocumentId, score, wifq.FirstPosition, wifq.QueryCount, docList.FirstPosition, docList.Count, i); docIdRank.Add(docList.DocumentId, docResult); } } docList = wifq.WordIndex.GetNext(); j++; if (j > wifq.WordIndex.Count) { break; } } } long maxScoreValue = 0; //Max score value of the docid that hit count less than wordIndexes.Length int wordIndexesLen = wordIndexes.Length; //Get the max score value of the docs that hit count less than wordIndexes.Length foreach (DocumentResultPoint docResult in docIdRank.Values) { if (docResult.pDocumentResult->HitCount < wordIndexesLen) { if (docResult.pDocumentResult->Score > maxScoreValue) { maxScoreValue = docResult.pDocumentResult->Score; } } } double hitRate = 0; if (indexInTop < wordIndexes.Length - 1) { int[] docidlist = new int[docIdRank.Count]; int i = 0; foreach (int docid in docIdRank.Keys) { docidlist[i] = docid; i++; } Array.Sort(docidlist); int lastWordHitCount = 0; foreach (int firstDocId in docidlist) { int curWord = indexInTop + 1; Core.SFQL.Parse.DocumentResultPoint drp; if (docIdRank.TryGetValue(firstDocId, out drp)) { while (curWord < wordIndexesLen) { Entity.DocumentPositionList docList = wordIndexes[curWord].WordIndex.Get(firstDocId); int curDocId = docList.DocumentId; if (curDocId >= 0) { drp.pDocumentResult->HitCount++; if (curWord == wordIndexesLen - 1) { lastWordHitCount++; } WordIndexForQuery wifq = wordIndexes[curWord]; long score = (long)wifq.FieldRank * (long)wifq.WordRank * (long)wifq.Idf_t * (long)docList.Count * (long)1000000 / ((long)wifq.Sum_d_t * (long)docList.TotalWordsInThisDocument); if (score < 0) { //Overflow score = long.MaxValue - 4000000; } else { switch (curWord) { case 0: score *= 20; break; case 1: score *= 4; break; case 2: score *= 1; break; case 3: score /= 2; break; default: score /= curWord; break; } } if (score < 0) { //Overflow score = long.MaxValue - 4000000; } drp.pDocumentResult->Score += score; double queryPositionDelta = wifq.FirstPosition - drp.pDocumentResult->LastWordIndexFirstPosition; double positionDelta = docList.FirstPosition - drp.pDocumentResult->LastPosition; double delta = Math.Abs(queryPositionDelta - positionDelta); if (delta < 0.031) { delta = 0.031; } else if (delta <= 1.1) { delta = 0.5; } else if (delta <= 2.1) { delta = 1; } delta = Math.Pow((1 / delta), ratio) * docList.Count * drp.pDocumentResult->LastCount / (double)(wifq.QueryCount * drp.pDocumentResult->LastWordIndexQueryCount); drp.pDocumentResult->Score = (long)(drp.pDocumentResult->Score * delta); //Overflow, if match too much, sometime score would less than zero. if (drp.pDocumentResult->Score < 0) { drp.pDocumentResult->Score = long.MaxValue - 4000000; } drp.pDocumentResult->LastIndex = (UInt16)curWord; drp.pDocumentResult->LastPosition = docList.FirstPosition; drp.pDocumentResult->LastCount = (UInt16)docList.Count; drp.pDocumentResult->LastWordIndexFirstPosition = (UInt16)wifq.FirstPosition; } curWord++; } //While if (drp.pDocumentResult->HitCount < wordIndexesLen) { if (drp.pDocumentResult->Score > maxScoreValue) { maxScoreValue = drp.pDocumentResult->Score; } } } } if (docidlist.Length > 0) { hitRate = (double)lastWordHitCount / (double)docidlist.Length; } } //Adjust score of the docs that hit count equal wordIndexes.Length foreach (DocumentResultPoint docResult in docIdRank.Values) { if (docResult.pDocumentResult->HitCount == wordIndexesLen) { docResult.pDocumentResult->Score += maxScoreValue; if (docResult.pDocumentResult->Score < 0) { docResult.pDocumentResult->Score = long.MaxValue; } } } performanceReport.Stop(); documentSum += docIdRank.Count; if (indexInTop < wordIndexes.Length - 1) { documentSum += wordIndexes[wordIndexes.Length - 1].RelTotalCount; if (hitRate > 0) { int predictCount = 0; for (int i = indexInTop + 1; i < wordIndexes.Length - 1; i++) { predictCount += (int)(wordIndexes[i].RelTotalCount * (1 - hitRate)); } documentSum += predictCount; } } if (documentSum > _TotalDocuments) { documentSum = _TotalDocuments; } docIdRank.RelTotalCount = documentSum; DeleteProvider delProvider = _DBProvider.DelProvider; int deleteCount = delProvider.Filter(docIdRank); docIdRank.RelTotalCount -= deleteCount; if (groupbyScanAll) { docIdRank.RelTotalCount = docIdRank.GroupByCollection.Count; } else if (docIdRank.GroupByCollection.Count > docIdRank.RelTotalCount) { docIdRank.RelTotalCount = docIdRank.GroupByCollection.Count; } }
public Core.SFQL.Parse.DocumentResultWhereDictionary Search() { Query.PerformanceReport performanceReport = new Hubble.Core.Query.PerformanceReport("Search of Match"); Init(); Core.SFQL.Parse.DocumentResultWhereDictionary result = new Core.SFQL.Parse.DocumentResultWhereDictionary(); if (_QueryWords.Count <= 0 || _WordIndexes.Length <= 0) { if (_QueryParameter.Not && UpDict != null) { return(UpDict); } else { return(result); } } if (this._QueryParameter.Not) { if (_InvertedIndex.IndexMode == Field.IndexMode.Simple) { Calculate(null, ref result, _WordIndexes); } else { //For not, we have to return all of the records //Modified at 18 Jan 2012 CalculateWithPosition(null, ref result, _WordIndexes); //if (Optimize.OptimizeArgument.IsOrderByScoreDesc(this._QueryParameter.OrderBys) && // !this._QueryParameter.NeedDistinct && this._QueryParameter.CanLoadPartOfDocs // && this._QueryParameter.AndExpressionCanBeOptimized(_DBProvider) // && _WordIndexes.Length > 1 && this.UpDict == null) //{ // CalculateWithPositionOrderByScoreDesc(null, ref result, _WordIndexes); //} //else //{ // CalculateWithPosition(null, ref result, _WordIndexes); //} } } else { if (_InvertedIndex.IndexMode == Field.IndexMode.Simple) { Calculate(this.UpDict, ref result, _WordIndexes); } else { if (//Optimize.OptimizeArgument.IsOrderByScoreDesc(this._QueryParameter.OrderBys) && !this._QueryParameter.NeedDistinct && this._QueryParameter.CanLoadPartOfDocs && this._QueryParameter.AndExpressionCanBeOptimized(_DBProvider) && _WordIndexes.Length > 1 && this.UpDict == null) { IQueryOptimize qOptimize = QueryOptimizeBuilder.Build(typeof(MatchOptimize), DBProvider, _QueryParameter.End, _QueryParameter.OrderBy, _QueryParameter.OrderBys, _QueryParameter.NeedGroupBy, _QueryParameter.OrderByCanBeOptimized, _QueryParameter.NeedFilterUntokenizedConditions(this._DBProvider), _QueryParameter.UntokenizedTreeOnRoot, _WordIndexes); Query.PerformanceReport performanceReportCalculate = null; try { performanceReportCalculate = new Hubble.Core.Query.PerformanceReport("Calculate"); qOptimize.CalculateOptimize(this.UpDict, ref result); } finally { performanceReportCalculate.Stop(); } } else { CalculateWithPosition(this.UpDict, ref result, _WordIndexes); } } } if (this._QueryParameter.Not) { result.Not = true; if (UpDict != null) { result = result.AndMergeForNot(result, UpDict); } } performanceReport.Stop(); return(result); }